Packages and utilities

library(tidyverse)
Registered S3 methods overwritten by 'dbplyr':
  method         from
  print.tbl_lazy     
  print.tbl_sql      
System has not been booted with systemd as init system (PID 1). Can't operate.
Failed to create bus connection: Host is down
running command 'timedatectl' had status 1── Attaching packages ─────────────────────────────────────────────────────────────────────────────────── tidyverse 1.3.0 ──
✓ ggplot2 3.3.3     ✓ purrr   0.3.4
✓ tibble  3.1.0     ✓ dplyr   1.0.5
✓ tidyr   1.1.3     ✓ stringr 1.4.0
✓ readr   1.4.0     ✓ forcats 0.5.1
── Conflicts ────────────────────────────────────────────────────────────────────────────────────── tidyverse_conflicts() ──
x dplyr::filter() masks stats::filter()
x dplyr::lag()    masks stats::lag()
library(lme4)
Loading required package: Matrix

Attaching package: ‘Matrix’

The following objects are masked from ‘package:tidyr’:

    expand, pack, unpack
library(lmerTest)

Attaching package: ‘lmerTest’

The following object is masked from ‘package:lme4’:

    lmer

The following object is masked from ‘package:stats’:

    step
library(logging)
library(mvtnorm)
library(mgcv)
Loading required package: nlme

Attaching package: ‘nlme’

The following object is masked from ‘package:lme4’:

    lmList

The following object is masked from ‘package:dplyr’:

    collapse

This is mgcv 1.8-34. For overview type 'help("mgcv-package")'.
# Provides bootstrap resampling tools
library(rsample)
# Compute the log-likelihood of a new dataset using a fit lme4 model.
logLik_test <- function(lm, test_X, test_y) {
  predictions <- predict(lm, test_X, re.form=NA)
  # Get std.dev. of residual, estimated from train data
  stdev <- sigma(lm)
  # For each prediction--observation, get the density p(obs | N(predicted, model_sigma)) and reduce
  density <- sum(dnorm(test_y, predictions, stdev, log=TRUE))
  return(density)
}
# Get per-prediction log-likelihood
logLik_test_per <- function(lm, test_X, test_y) {
  predictions <- predict(lm, test_X, re.form=NA)
  # Get std.dev. of residual, estimated from train data
  stdev <- sigma(lm)
  # For each prediction--observation, get the density p(obs | N(predicted, model_sigma))
  densities <- dnorm(test_y, predictions, stdev, log=TRUE)
  return(densities)
}
# Compute MSE of a new dataset using a fit lme4 model.
mse_test <- function(lm, test_X, test_y) {
  return(mean((predict(lm, test_X, re.form=NA) - test_y) ^ 2))
}
#Sanity checks
#mylm <- gam(psychometric ~  s(surprisal, bs = "cr", k = 20) + s(prev_surp, bs = "cr", k = 20) + te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr"), data=train_data)
#c(logLik(mylm), logLik_test(mylm, train_data, train_data$psychometric))
#logLik_test(mylm, test_data, test_data$psychometric)

Data loading and preprocessing

data = read.csv("../data/harmonized_results.csv")

all_data = data %>%
  mutate(seed = as.factor(seed)) %>%
  group_by(corpus, model, training, seed) %>%
    mutate(prev_surp = lag(surprisal),
         prev_code = lag(code),
         prev_len = lag(len),
         prev_freq = lag(freq),
         prev_surp = lag(surprisal),
         
         prev2_freq = lag(prev_freq),
         prev2_code = lag(prev_code),
         prev2_len = lag(prev_len),
         prev2_surp = lag(prev_surp),
         
         prev3_freq = lag(prev2_freq),
         prev3_code = lag(prev2_code),
         prev3_len = lag(prev2_len),
         prev3_surp = lag(prev2_surp),
         
         prev4_freq = lag(prev3_freq),
         prev4_code = lag(prev3_code),
         prev4_len = lag(prev3_len),
         prev4_surp = lag(prev3_surp)) %>%
  ungroup() %>%
  
  # Filter back two for the dundee corpus. Filter back 1 for all other corpora
  # NB this effectively removes all zero-surprisal rows, since early-sentence tokens don't have contiguous token history
  filter((corpus == "dundee" & code == prev2_code + 2) | (corpus != "dundee" & code == prev4_code + 4)) %>%
  
  select(-prev_code, -prev2_code, -prev3_code) %>%
  drop_na()

all_data = all_data %>%
  mutate(
    model = as.character(model),
    model = if_else(model == "gpt-2", "gpt2", model),
    model = as.factor(model))
missing_rows = all_data %>% complete(nesting(corpus, code), nesting(model, training, seed)) %>% 
  group_by(corpus, code) %>% 
    filter(sum(is.na(surprisal)) > 0) %>% 
  ungroup() %>% 
  anti_join(all_data, by=c("corpus", "code", "model", "training", "seed"))

missing_rows %>% ggplot(aes(x=corpus, fill=factor(paste(model,training)))) +
geom_bar(position=position_dodge(width=0.8))

print(missing_rows %>% group_by(model, training, seed, corpus) %>% summarise(n=n()) %>% arrange(desc(n)))
`summarise()` has grouped output by 'model', 'training', 'seed'. You can override using the `.groups` argument.

# Compute the ideal number of model--seed--training observations per token.
to_drop = all_data %>%
  group_by(corpus, code) %>% summarise(n = n()) %>% ungroup() %>%
  group_by(corpus) %>% mutate( max_n = max(n)) %>% ungroup() %>%
  filter(max_n != n) %>% 
  select(code, corpus)
`summarise()` has grouped output by 'corpus'. You can override using the `.groups` argument.
# Find tokens which have 28 observations and compare model+training freqs
all_data %>% filter(corpus == "bnc-brown") %>% group_by(code) %>% filter(n() == 28)
tempx = all_data %>% filter(corpus == "bnc-brown", code == 17103)
table(paste(tempx$model, tempx$training))

      5gram bllip-lg       5gram bllip-md       5gram bllip-sm       5gram bllip-xs        gpt2 bllip-lg 
                   1                    1                    1                    1                    1 
gpt2 bllip-lg-gptbpe        gpt2 bllip-md gpt2 bllip-md-gptbpe gpt2 bllip-sm-gptbpe gpt2 bllip-xs-gptbpe 
                   2                    1                    3                    1                    1 
       rnng bllip-lg        rnng bllip-md        rnng bllip-sm        rnng bllip-xs     vanilla bllip-lg 
                   1                    2                    2                    4                    1 
    vanilla bllip-md     vanilla bllip-sm     vanilla bllip-xs 
                   3                    3                    3 
# This one is missing a seed
#to_drop %>% filter(corpus == "bnc-brown") %>% arrange(code)
tempx = all_data %>% filter(corpus == "bnc-brown", code == 17017)
table(paste(tempx$model, tempx$training))

      5gram bllip-lg       5gram bllip-md       5gram bllip-sm       5gram bllip-xs        gpt2 bllip-lg 
                   1                    1                    1                    1                    1 
gpt2 bllip-lg-gptbpe        gpt2 bllip-md gpt2 bllip-md-gptbpe gpt2 bllip-sm-gptbpe gpt2 bllip-xs-gptbpe 
                   2                    1                    3                    1                    1 
       rnng bllip-lg        rnng bllip-md        rnng bllip-sm        rnng bllip-xs     vanilla bllip-lg 
                   1                    2                    2                    4                    1 
    vanilla bllip-md     vanilla bllip-sm     vanilla bllip-xs 
                   3                    3                    3 
# nvm somehow not a problem anymore ..
# # Zooming in on the problem -- why is there no bllip-lg data here?
# all_data %>% filter(corpus == "dundee", model == "vanilla", training == "bllip-lg", code > 10720, code < 10730)

loginfo(paste("Dropping", nrow(to_drop), "observations corresponding to corpus tokens which are missing observations for some model."))
2021-04-09 18:04:10 INFO::Dropping 10561 observations corresponding to corpus tokens which are missing observations for some model.
loginfo(paste("Dropping", to_drop %>% group_by(corpus, code) %>% n_groups(), "tokens which are missing observations for some model."))
2021-04-09 18:04:10 INFO::Dropping 10561 tokens which are missing observations for some model.
all_data = all_data %>% anti_join(to_drop %>% group_by(corpus, code), by=c("corpus", "code"))
loginfo(paste("After drop,", nrow(all_data), "observations (", all_data %>% group_by(corpus, code) %>% n_groups(), " tokens) remain."))
2021-04-09 18:04:11 INFO::After drop, 1056239 observations ( 33115  tokens) remain.

to_drop_zero_surps = all_data %>% group_by(corpus, code) %>% filter(any(surprisal == 0)) %>% ungroup()
loginfo(paste("Dropping", nrow(to_drop_zero_surps), "observations corresponding to corpus tokens which have surprisal zeros for some model."))
2021-04-09 18:04:11 INFO::Dropping 128 observations corresponding to corpus tokens which have surprisal zeros for some model.
loginfo(paste("Dropping", to_drop_zero_surps %>% group_by(corpus, code) %>% n_groups(), "tokens which have surprisal zeros for some model."))
2021-04-09 18:04:11 INFO::Dropping 4 tokens which have surprisal zeros for some model.
all_data = all_data %>% anti_join(to_drop_zero_surps %>% group_by(corpus, code), by=c("corpus", "code"))
loginfo(paste("After drop,", nrow(all_data), "observations (", all_data %>% group_by(corpus, code) %>% n_groups(), " tokens) remain."))
2021-04-09 18:04:11 INFO::After drop, 1056111 observations ( 33111  tokens) remain.

to_drop_zero_psychs = all_data %>% group_by(corpus, code) %>% filter(any(psychometric == 0)) %>% ungroup()
loginfo(paste("Dropping", nrow(to_drop_zero_psychs), "observations corresponding to corpus tokens which have psychometric zeros for some model."))
2021-04-09 18:04:12 INFO::Dropping 16480 observations corresponding to corpus tokens which have psychometric zeros for some model.
loginfo(paste("Dropping", to_drop_zero_psychs %>% group_by(corpus, code) %>% n_groups(), "tokens which have psychometric zeros for some model."))
2021-04-09 18:04:12 INFO::Dropping 515 tokens which have psychometric zeros for some model.
all_data = all_data %>% anti_join(to_drop_zero_psychs %>% group_by(corpus, code), by=c("corpus", "code"))
loginfo(paste("After drop,", nrow(all_data), "observations (", all_data %>% group_by(corpus, code) %>% n_groups(), " tokens) remain."))
2021-04-09 18:04:12 INFO::After drop, 1039631 observations ( 32596  tokens) remain.

Train Linear Models which are used to assess Delta Log Lik

# Compute linear model stats for the given training data subset and full test data.
# Automatically subsets the test data to match the relevant group for which we are training a linear model.
get_lm_data <- function(df, test_data, formula, fold, store_env) {
  #this_lm <- gam(formula, data=df);
  print(paste(unique(paste(df$model, df$training, df$seed, df$corpus))[1], fold, nrow(df)))
  this_lm = lm(formula, data=df)
  this_test_data <- semi_join(test_data, df, by=c("training", "model", "seed", "corpus"));
  
  # Save lm to the global env so that we can access residuals later.
  lm_name = paste(unique(paste(df$model, df$training, df$seed, df$corpus))[1], fold)
  assign(lm_name, this_lm, envir=store_env)
  
  summarise(df,
            log_lik = as.numeric(logLik(this_lm, REML = F)),
            test_lik = logLik_test(this_lm, this_test_data, this_test_data$psychometric),
            test_mse = mse_test(this_lm, this_test_data, this_test_data$psychometric))
}
# For a previously fitted lm stored in store_env, get the residuals on test data of the relevant data subset.
get_lm_residuals <- function(df, fold, store_env) {
  # Retrieve the relevant lm.
  lm_name = paste(unique(paste(df$model, df$training, df$seed, df$corpus))[1], fold)
  this_lm <- get(lm_name, envir=store_env)
  
  mutate(df,
         likelihood = logLik_test_per(this_lm, df, df$psychometric),
         resid = df$psychometric - predict(this_lm, df, re.form=NA))
}
# Compute per-example delta-log-likelihood for the given test fold.
get_lm_delta_log_lik <- function(test_data, fold, baseline_env, full_env) {
  lm_name = paste(unique(paste(test_data$model, test_data$training, test_data$seed, test_data$corpus))[1], fold)
  baseline_lm <- get(lm_name, envir=baseline_env)
  full_lm <- get(lm_name, envir=full_env)
  
  delta_log_lik = logLik_test_per(full_lm, test_data, test_data$psychometric) - logLik_test_per(baseline_lm, test_data, test_data$psychometric)
  return(cbind(test_data, delta_log_lik=delta_log_lik))
}
#####
# Define regression formulae.

# Regression code to fit GAM models.
#baseline_rt_regression = psychometric ~ te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr") + te(prev2_freq, prev2_len, bs = "cr")
#baselie_sprt_regression = psychometric ~ te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr") + te(prev2_freq, prev2_len, bs = "cr") + te(prev3_freq, prev3_len, bs = "cr") + te(prev4_freq, prev4_len, bs = "cr")

#full_rt_regression = psychometric ~ s(surprisal, bs = "cr", k = 20) + s(prev_surp, bs = "cr", k = 20) + s(prev2_surp, bs = "cr", k = 20) + te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr") + te(prev2_freq, prev2_len, bs = "cr")
#full_sprt_regression = psychometric ~ s(surprisal, bs = "cr", k = 20) + s(prev_surp, bs = "cr", k = 20) + s(prev2_surp, bs = "cr", k = 20) + s(prev3_surp, bs = "cr", k = 20) + s(prev4_surp, bs = "cr", k = 20) + te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr") + te(prev2_freq, prev2_len, bs = "cr") + te(prev3_freq, prev3_len, bs = "cr") + te(prev4_freq, prev4_len, bs = "cr")

# Regression Code to fit linear models
baseline_rt_regression = psychometric ~ freq + prev_freq + prev2_freq + len + prev_len + prev2_len
baseline_sprt_regression = psychometric ~ freq + prev_freq + prev2_freq + prev3_freq + prev4_freq + len + prev_len + prev2_len + prev3_len + prev4_len

full_sprt_regression = psychometric ~ surprisal + prev_surp + prev2_surp + prev3_surp + prev4_surp + freq + prev_freq + prev2_freq + prev3_freq + prev4_freq + len + prev_len + prev2_len + prev3_len + prev4_len
full_rt_regression = psychometric ~ surprisal + prev_surp + prev2_surp + freq + prev_freq + prev2_freq + len + prev_len + prev2_len
  
#####
# Prepare frames/environments for storing results/objects.
baseline_results = data.frame()
full_model_results = data.frame()
baseline_residuals = data.frame()
full_residuals = data.frame()
log_lik_deltas = data.frame()

#Randomly shuffle the data
all_data<-all_data[sample(nrow(all_data)),]
#Create K equally size folds
K = 10
folds <- cut(seq(1,nrow(all_data)),breaks=K,labels=FALSE)
#Perform 10 fold cross validation

# Fit models for some fold of the data.
baseline_corpus = function(corpus, df, test_data, fold, env) {
  if(corpus == "dundee") {
    get_lm_data(df, test_data, baseline_rt_regression, fold, env)
  } else {
    get_lm_data(df, test_data, baseline_sprt_regression, fold, env)
  }
}
full_model_corpus = function(corpus, df, test_data, fold, env) {
  if(corpus[1] == "dundee") {
    get_lm_data(df, test_data, full_rt_regression, fold, env)
  } else {
    get_lm_data(df, test_data, full_sprt_regression, fold, env)
  }
}

# Prepare a new Environment in which we store fitted LMs, which we'll query later for residuals and other metrics.
baseline_env = new.env()
full_env = new.env()

for(i in 1:K) { 
  #Segement your data by fold using the which() function 
  testIndexes <- which(folds==i, arr.ind=TRUE)
  test_data <- all_data[testIndexes, ]
  train_data <- all_data[-testIndexes, ]
  
  # Compute a baseline linear model for each model--training--seed--RT-corpus combination.
  baselines = train_data %>%
    group_by(model, training, seed, corpus) %>%
      print(model) %>%
      do(baseline_corpus(unique(.$corpus), ., test_data, i, baseline_env)) %>%
    ungroup() %>%
    mutate(seed = as.factor(seed),
           fold = i)
  
  baseline_results = rbind(baseline_results, baselines)
  
  # Compute a full linear model for each model--training--seed-RT-corpus combination
  full_models = train_data %>%
    group_by(model, training, seed, corpus) %>%
      do(full_model_corpus(unique(.$corpus), ., test_data, i, full_env)) %>%
    ungroup() %>%
    mutate(seed = as.factor(seed),
           fold = i)
  
  full_model_results = rbind(full_model_results, full_models)
  
  # Compute delta-log-likelihoods
  fold_log_lik_deltas = test_data %>%
    group_by(model, training, seed, corpus) %>%
      do(get_lm_delta_log_lik(., i, baseline_env, full_env)) %>%
    ungroup()

  log_lik_deltas = rbind(log_lik_deltas, fold_log_lik_deltas)
  
  fold_baseline_residuals = test_data %>%
    group_by(model, training, seed, corpus) %>%
      do(get_lm_residuals(., i, baseline_env)) %>%
    ungroup()

  baseline_residuals = rbind(baseline_residuals, fold_baseline_residuals)

  fold_full_residuals = test_data %>%
    group_by(model, training, seed, corpus) %>%
      do(get_lm_residuals(., i, full_env)) %>%
    ungroup()

  full_residuals = rbind(full_residuals, fold_full_residuals)
}
[1] "5gram bllip-lg 1111 bnc-brown 1 1700"
[1] "5gram bllip-lg 1111 dundee 1 24500"
[1] "5gram bllip-lg 1111 natural-stories 1 3134"
[1] "5gram bllip-md 1111 bnc-brown 1 1724"
[1] "5gram bllip-md 1111 dundee 1 24539"
[1] "5gram bllip-md 1111 natural-stories 1 3067"
[1] "5gram bllip-sm 1111 bnc-brown 1 1684"
[1] "5gram bllip-sm 1111 dundee 1 24567"
[1] "5gram bllip-sm 1111 natural-stories 1 3118"
[1] "5gram bllip-xs 1111 bnc-brown 1 1664"
[1] "5gram bllip-xs 1111 dundee 1 24576"
[1] "5gram bllip-xs 1111 natural-stories 1 3112"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 1 1720"
[1] "gpt2 bllip-lg 1587139950 dundee 1 24502"
[1] "gpt2 bllip-lg 1587139950 natural-stories 1 3096"
[1] "gpt2 bllip-lg-gptbpe 1581955288 bnc-brown 1 1683"
[1] "gpt2 bllip-lg-gptbpe 1581955288 dundee 1 24485"
[1] "gpt2 bllip-lg-gptbpe 1581955288 natural-stories 1 3106"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 1 1698"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 1 24566"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 1 3074"
[1] "gpt2 bllip-md 1586986276 bnc-brown 1 1681"
[1] "gpt2 bllip-md 1586986276 dundee 1 24575"
[1] "gpt2 bllip-md 1586986276 natural-stories 1 3086"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 1 24479"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 1 3100"
[1] "gpt2 bllip-md-gptbpe 1581861474 bnc-brown 1 1682"
[1] "gpt2 bllip-md-gptbpe 1581861474 dundee 1 24547"
[1] "gpt2 bllip-md-gptbpe 1581861474 natural-stories 1 3113"
[1] "gpt2 bllip-md-gptbpe 1582126320 bnc-brown 1 1691"
[1] "gpt2 bllip-md-gptbpe 1582126320 dundee 1 24516"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 1 1684"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 1 24526"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 1 3078"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 1 1690"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 1 24589"

|=====================================                                                             | 38% ~3 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 1 3083"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 1 1689"

|=======================================                                                           | 40% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 1 24598"

|========================================                                                          | 41% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 1 3110"
[1] "rnng bllip-lg 7245 bnc-brown 1 1692"

|==========================================                                                        | 43% ~3 s remaining     [1] "rnng bllip-lg 7245 dundee 1 24580"

|===========================================                                                       | 44% ~3 s remaining     [1] "rnng bllip-lg 7245 natural-stories 1 3074"
[1] "rnng bllip-md 3602 bnc-brown 1 1714"

|=============================================                                                     | 46% ~3 s remaining     [1] "rnng bllip-md 3602 dundee 1 24561"

|==============================================                                                    | 47% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 1 3090"
[1] "rnng bllip-md 44862 bnc-brown 1 1705"

|================================================                                                  | 49% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 1 24525"

|=================================================                                                 | 51% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 1 3079"
[1] "rnng bllip-sm 7877 bnc-brown 1 1689"

|===================================================                                               | 53% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 1 24536"

|====================================================                                              | 54% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 1 3112"
[1] "rnng bllip-sm 64924 bnc-brown 1 1701"

|======================================================                                            | 56% ~2 s remaining     [1] "rnng bllip-sm 64924 dundee 1 24561"

|=======================================================                                           | 57% ~2 s remaining     [1] "rnng bllip-sm 64924 natural-stories 1 3101"
[1] "rnng bllip-xs 4301 bnc-brown 1 1694"

|=========================================================                                         | 59% ~2 s remaining     [1] "rnng bllip-xs 4301 dundee 1 24515"

|==========================================================                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 1 3098"
[1] "rnng bllip-xs 28066 bnc-brown 1 1702"

|============================================================                                      | 62% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 1 24511"

|=============================================================                                     | 63% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 1 3085"
[1] "rnng bllip-xs 28068 bnc-brown 1 1724"

|===============================================================                                   | 65% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 1 24518"

|================================================================                                  | 66% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 1 3119"
[1] "rnng bllip-xs 51272 bnc-brown 1 1705"

|===================================================================                               | 68% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 1 24588"

|====================================================================                              | 69% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 1 3084"
[1] "vanilla bllip-lg 111 bnc-brown 1 1681"

|======================================================================                            | 72% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 1 24605"

|=======================================================================                           | 73% ~1 s remaining     [1] "vanilla bllip-lg 111 natural-stories 1 3091"
[1] "vanilla bllip-md 120 bnc-brown 1 1727"

|=========================================================================                         | 75% ~1 s remaining     [1] "vanilla bllip-md 120 dundee 1 24578"

|==========================================================================                        | 76% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 1 3095"
[1] "vanilla bllip-md 607 bnc-brown 1 1686"

|============================================================================                      | 78% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 1 1710"
[1] "vanilla bllip-md 922 dundee 1 24491"

|==============================================================================                    | 80% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 1 3110"
[1] "vanilla bllip-sm 111 bnc-brown 1 1701"

|================================================================================                  | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 1 24641"

|=================================================================================                 | 83% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 1 3096"
[1] "vanilla bllip-sm 120 bnc-brown 1 1681"

|===================================================================================               | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 1 24565"

|====================================================================================              | 86% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 1 3063"
[1] "vanilla bllip-sm 922 bnc-brown 1 1678"

|======================================================================================            | 88% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 1 24565"

|=======================================================================================           | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 1 3127"
[1] "vanilla bllip-xs 111 bnc-brown 1 1665"

|=========================================================================================         | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 dundee 1 24604"

|==========================================================================================        | 93% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 1 3080"
[1] "vanilla bllip-xs 120 bnc-brown 1 1678"

|============================================================================================      | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 1 24534"

|=============================================================================================     | 96% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 1 3070"
[1] "vanilla bllip-xs 922 bnc-brown 1 1670"

|===============================================================================================   | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 1 24491"

|================================================================================================  | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 1 3089"
[1] "5gram bllip-lg 1111 bnc-brown 1 1700"
[1] "5gram bllip-lg 1111 dundee 1 24500"
[1] "5gram bllip-lg 1111 natural-stories 1 3134"
[1] "5gram bllip-md 1111 bnc-brown 1 1724"
[1] "5gram bllip-md 1111 dundee 1 24539"
[1] "5gram bllip-md 1111 natural-stories 1 3067"
[1] "5gram bllip-sm 1111 bnc-brown 1 1684"
[1] "5gram bllip-sm 1111 dundee 1 24567"
[1] "5gram bllip-sm 1111 natural-stories 1 3118"
[1] "5gram bllip-xs 1111 bnc-brown 1 1664"
[1] "5gram bllip-xs 1111 dundee 1 24576"
[1] "5gram bllip-xs 1111 natural-stories 1 3112"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 1 1720"
[1] "gpt2 bllip-lg 1587139950 dundee 1 24502"
[1] "gpt2 bllip-lg 1587139950 natural-stories 1 3096"
[1] "gpt2 bllip-lg-gptbpe 1581955288 bnc-brown 1 1683"
[1] "gpt2 bllip-lg-gptbpe 1581955288 dundee 1 24485"
[1] "gpt2 bllip-lg-gptbpe 1581955288 natural-stories 1 3106"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 1 1698"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 1 24566"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 1 3074"
[1] "gpt2 bllip-md 1586986276 bnc-brown 1 1681"
[1] "gpt2 bllip-md 1586986276 dundee 1 24575"
[1] "gpt2 bllip-md 1586986276 natural-stories 1 3086"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 1 24479"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 1 3100"
[1] "gpt2 bllip-md-gptbpe 1581861474 bnc-brown 1 1682"
[1] "gpt2 bllip-md-gptbpe 1581861474 dundee 1 24547"
[1] "gpt2 bllip-md-gptbpe 1581861474 natural-stories 1 3113"
[1] "gpt2 bllip-md-gptbpe 1582126320 bnc-brown 1 1691"
[1] "gpt2 bllip-md-gptbpe 1582126320 dundee 1 24516"

|===============================                                                                   | 33% ~5 s remaining     [1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 1 1684"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 1 24526"

|==================================                                                                | 35% ~5 s remaining     [1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 1 3078"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 1 1690"

|====================================                                                              | 37% ~4 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 1 24589"

|=====================================                                                             | 38% ~4 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 1 3083"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 1 1689"

|=======================================                                                           | 40% ~4 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 1 24598"

|========================================                                                          | 41% ~4 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 1 3110"
[1] "rnng bllip-lg 7245 bnc-brown 1 1692"

|==========================================                                                        | 43% ~4 s remaining     [1] "rnng bllip-lg 7245 dundee 1 24580"

|===========================================                                                       | 44% ~4 s remaining     [1] "rnng bllip-lg 7245 natural-stories 1 3074"
[1] "rnng bllip-md 3602 bnc-brown 1 1714"

|=============================================                                                     | 46% ~4 s remaining     [1] "rnng bllip-md 3602 dundee 1 24561"

|==============================================                                                    | 47% ~4 s remaining     [1] "rnng bllip-md 3602 natural-stories 1 3090"
[1] "rnng bllip-md 44862 bnc-brown 1 1705"

|================================================                                                  | 49% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 1 24525"

|=================================================                                                 | 51% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 1 3079"
[1] "rnng bllip-sm 7877 bnc-brown 1 1689"

|===================================================                                               | 53% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 1 24536"

|====================================================                                              | 54% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 1 3112"
[1] "rnng bllip-sm 64924 bnc-brown 1 1701"

|======================================================                                            | 56% ~3 s remaining     [1] "rnng bllip-sm 64924 dundee 1 24561"

|=======================================================                                           | 57% ~3 s remaining     [1] "rnng bllip-sm 64924 natural-stories 1 3101"

|========================================================                                          | 58% ~3 s remaining     [1] "rnng bllip-xs 4301 bnc-brown 1 1694"
[1] "rnng bllip-xs 4301 dundee 1 24515"

|==========================================================                                        | 60% ~3 s remaining     [1] "rnng bllip-xs 4301 natural-stories 1 3098"
[1] "rnng bllip-xs 28066 bnc-brown 1 1702"

|============================================================                                      | 62% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 1 24511"

|=============================================================                                     | 63% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 1 3085"
[1] "rnng bllip-xs 28068 bnc-brown 1 1724"

|===============================================================                                   | 65% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 1 24518"

|================================================================                                  | 66% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 1 3119"
[1] "rnng bllip-xs 51272 bnc-brown 1 1705"

|===================================================================                               | 68% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 1 24588"

|====================================================================                              | 69% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 1 3084"
[1] "vanilla bllip-lg 111 bnc-brown 1 1681"

|======================================================================                            | 72% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 1 24605"

|=======================================================================                           | 73% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 1 3091"
[1] "vanilla bllip-md 120 bnc-brown 1 1727"

|=========================================================================                         | 75% ~2 s remaining     [1] "vanilla bllip-md 120 dundee 1 24578"

|==========================================================================                        | 76% ~2 s remaining     [1] "vanilla bllip-md 120 natural-stories 1 3095"
[1] "vanilla bllip-md 607 bnc-brown 1 1686"

|============================================================================                      | 78% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 1 1710"
[1] "vanilla bllip-md 922 dundee 1 24491"

|==============================================================================                    | 80% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 1 3110"
[1] "vanilla bllip-sm 111 bnc-brown 1 1701"

|================================================================================                  | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 1 24641"

|=================================================================================                 | 83% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 1 3096"
[1] "vanilla bllip-sm 120 bnc-brown 1 1681"

|===================================================================================               | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 1 24565"

|====================================================================================              | 86% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 1 3063"
[1] "vanilla bllip-sm 922 bnc-brown 1 1678"

|======================================================================================            | 88% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 1 24565"

|=======================================================================================           | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 1 3127"
[1] "vanilla bllip-xs 111 bnc-brown 1 1665"

|=========================================================================================         | 92% ~1 s remaining     [1] "vanilla bllip-xs 111 dundee 1 24604"

|==========================================================================================        | 93% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 1 3080"
[1] "vanilla bllip-xs 120 bnc-brown 1 1678"

|============================================================================================      | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 1 24534"

|=============================================================================================     | 96% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 1 3070"
[1] "vanilla bllip-xs 922 bnc-brown 1 1670"

|===============================================================================================   | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 1 24491"

|================================================================================================  | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 1 3089"

|==================================================================================================|100% ~0 s remaining     
[1] "5gram bllip-lg 1111 bnc-brown 2 1690"
[1] "5gram bllip-lg 1111 dundee 2 24511"
[1] "5gram bllip-lg 1111 natural-stories 2 3102"
[1] "5gram bllip-md 1111 bnc-brown 2 1667"
[1] "5gram bllip-md 1111 dundee 2 24509"
[1] "5gram bllip-md 1111 natural-stories 2 3093"
[1] "5gram bllip-sm 1111 bnc-brown 2 1686"
[1] "5gram bllip-sm 1111 dundee 2 24533"
[1] "5gram bllip-sm 1111 natural-stories 2 3094"
[1] "5gram bllip-xs 1111 bnc-brown 2 1692"
[1] "5gram bllip-xs 1111 dundee 2 24599"
[1] "5gram bllip-xs 1111 natural-stories 2 3099"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 2 1701"
[1] "gpt2 bllip-lg 1587139950 dundee 2 24492"
[1] "gpt2 bllip-lg 1587139950 natural-stories 2 3073"
[1] "gpt2 bllip-lg-gptbpe 1581955288 bnc-brown 2 1684"
[1] "gpt2 bllip-lg-gptbpe 1581955288 dundee 2 24573"
[1] "gpt2 bllip-lg-gptbpe 1581955288 natural-stories 2 3094"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 2 1708"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 2 24522"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 2 3100"
[1] "gpt2 bllip-md 1586986276 bnc-brown 2 1693"
[1] "gpt2 bllip-md 1586986276 dundee 2 24514"
[1] "gpt2 bllip-md 1586986276 natural-stories 2 3112"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 2 24674"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 2 3092"
[1] "gpt2 bllip-md-gptbpe 1581861474 bnc-brown 2 1698"
[1] "gpt2 bllip-md-gptbpe 1581861474 dundee 2 24481"

|============================                                                                      | 29% ~5 s remaining     [1] "gpt2 bllip-md-gptbpe 1581861474 natural-stories 2 3087"
[1] "gpt2 bllip-md-gptbpe 1582126320 bnc-brown 2 1719"

|==============================                                                                    | 32% ~5 s remaining     [1] "gpt2 bllip-md-gptbpe 1582126320 dundee 2 24576"

|===============================                                                                   | 33% ~4 s remaining     [1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 2 1669"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 2 24536"

|==================================                                                                | 35% ~4 s remaining     [1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 2 3103"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 2 1679"

|====================================                                                              | 37% ~4 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 2 24480"

|=====================================                                                             | 38% ~4 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 2 3105"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 2 1699"

|=======================================                                                           | 40% ~4 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 2 24458"

|========================================                                                          | 41% ~4 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 2 3095"
[1] "rnng bllip-lg 7245 bnc-brown 2 1677"

|==========================================                                                        | 43% ~4 s remaining     [1] "rnng bllip-lg 7245 dundee 2 24521"

|===========================================                                                       | 44% ~4 s remaining     [1] "rnng bllip-lg 7245 natural-stories 2 3080"
[1] "rnng bllip-md 3602 bnc-brown 2 1676"

|=============================================                                                     | 46% ~3 s remaining     [1] "rnng bllip-md 3602 dundee 2 24644"

|==============================================                                                    | 47% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 2 3097"
[1] "rnng bllip-md 44862 bnc-brown 2 1709"

|================================================                                                  | 49% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 2 24495"

|=================================================                                                 | 51% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 2 3122"
[1] "rnng bllip-sm 7877 bnc-brown 2 1710"

|===================================================                                               | 53% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 2 24550"

|====================================================                                              | 54% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 2 3084"
[1] "rnng bllip-sm 64924 bnc-brown 2 1701"

|======================================================                                            | 56% ~3 s remaining     [1] "rnng bllip-sm 64924 dundee 2 24605"

|=======================================================                                           | 57% ~3 s remaining     [1] "rnng bllip-sm 64924 natural-stories 2 3118"
[1] "rnng bllip-xs 4301 bnc-brown 2 1696"

|=========================================================                                         | 59% ~2 s remaining     [1] "rnng bllip-xs 4301 dundee 2 24544"

|==========================================================                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 2 3100"
[1] "rnng bllip-xs 28066 bnc-brown 2 1687"

|============================================================                                      | 62% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 2 24542"

|=============================================================                                     | 63% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 2 3101"
[1] "rnng bllip-xs 28068 bnc-brown 2 1693"

|===============================================================                                   | 65% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 2 24518"

|================================================================                                  | 66% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 2 3100"
[1] "rnng bllip-xs 51272 bnc-brown 2 1699"

|===================================================================                               | 68% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 2 24596"

|====================================================================                              | 69% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 2 3082"
[1] "vanilla bllip-lg 111 bnc-brown 2 1690"

|======================================================================                            | 72% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 2 24587"

|=======================================================================                           | 73% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 2 3121"
[1] "vanilla bllip-md 120 bnc-brown 2 1670"

|=========================================================================                         | 75% ~2 s remaining     [1] "vanilla bllip-md 120 dundee 2 24593"

|==========================================================================                        | 76% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 2 3075"
[1] "vanilla bllip-md 607 bnc-brown 2 1691"

|============================================================================                      | 78% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 2 1686"
[1] "vanilla bllip-md 922 dundee 2 24550"

|==============================================================================                    | 80% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 2 3093"
[1] "vanilla bllip-sm 111 bnc-brown 2 1686"

|================================================================================                  | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 2 24476"

|=================================================================================                 | 83% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 2 3104"

|==================================================================================                | 84% ~1 s remaining     [1] "vanilla bllip-sm 120 bnc-brown 2 1703"
[1] "vanilla bllip-sm 120 dundee 2 24594"

|====================================================================================              | 86% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 2 3110"
[1] "vanilla bllip-sm 922 bnc-brown 2 1688"

|======================================================================================            | 88% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 2 24486"

|=======================================================================================           | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 2 3118"
[1] "vanilla bllip-xs 111 bnc-brown 2 1692"

|=========================================================================================         | 92% ~1 s remaining     [1] "vanilla bllip-xs 111 dundee 2 24555"

|==========================================================================================        | 93% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 2 3127"
[1] "vanilla bllip-xs 120 bnc-brown 2 1689"

|============================================================================================      | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 2 24591"

|=============================================================================================     | 96% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 2 3072"
[1] "vanilla bllip-xs 922 bnc-brown 2 1708"

|===============================================================================================   | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 2 24569"

|================================================================================================  | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 2 3105"
[1] "5gram bllip-lg 1111 bnc-brown 2 1690"
[1] "5gram bllip-lg 1111 dundee 2 24511"
[1] "5gram bllip-lg 1111 natural-stories 2 3102"
[1] "5gram bllip-md 1111 bnc-brown 2 1667"
[1] "5gram bllip-md 1111 dundee 2 24509"
[1] "5gram bllip-md 1111 natural-stories 2 3093"
[1] "5gram bllip-sm 1111 bnc-brown 2 1686"
[1] "5gram bllip-sm 1111 dundee 2 24533"
[1] "5gram bllip-sm 1111 natural-stories 2 3094"
[1] "5gram bllip-xs 1111 bnc-brown 2 1692"
[1] "5gram bllip-xs 1111 dundee 2 24599"
[1] "5gram bllip-xs 1111 natural-stories 2 3099"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 2 1701"
[1] "gpt2 bllip-lg 1587139950 dundee 2 24492"
[1] "gpt2 bllip-lg 1587139950 natural-stories 2 3073"
[1] "gpt2 bllip-lg-gptbpe 1581955288 bnc-brown 2 1684"
[1] "gpt2 bllip-lg-gptbpe 1581955288 dundee 2 24573"
[1] "gpt2 bllip-lg-gptbpe 1581955288 natural-stories 2 3094"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 2 1708"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 2 24522"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 2 3100"
[1] "gpt2 bllip-md 1586986276 bnc-brown 2 1693"
[1] "gpt2 bllip-md 1586986276 dundee 2 24514"
[1] "gpt2 bllip-md 1586986276 natural-stories 2 3112"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 2 24674"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 2 3092"
[1] "gpt2 bllip-md-gptbpe 1581861474 bnc-brown 2 1698"
[1] "gpt2 bllip-md-gptbpe 1581861474 dundee 2 24481"
[1] "gpt2 bllip-md-gptbpe 1581861474 natural-stories 2 3087"
[1] "gpt2 bllip-md-gptbpe 1582126320 bnc-brown 2 1719"
[1] "gpt2 bllip-md-gptbpe 1582126320 dundee 2 24576"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 2 1669"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 2 24536"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 2 3103"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 2 1679"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 2 24480"

|=====================================                                                             | 38% ~3 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 2 3105"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 2 1699"

|=======================================                                                           | 40% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 2 24458"

|========================================                                                          | 41% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 2 3095"
[1] "rnng bllip-lg 7245 bnc-brown 2 1677"

|==========================================                                                        | 43% ~3 s remaining     [1] "rnng bllip-lg 7245 dundee 2 24521"

|===========================================                                                       | 44% ~3 s remaining     [1] "rnng bllip-lg 7245 natural-stories 2 3080"
[1] "rnng bllip-md 3602 bnc-brown 2 1676"

|=============================================                                                     | 46% ~3 s remaining     [1] "rnng bllip-md 3602 dundee 2 24644"

|==============================================                                                    | 47% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 2 3097"
[1] "rnng bllip-md 44862 bnc-brown 2 1709"

|================================================                                                  | 49% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 2 24495"

|=================================================                                                 | 51% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 2 3122"
[1] "rnng bllip-sm 7877 bnc-brown 2 1710"

|===================================================                                               | 53% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 2 24550"

|====================================================                                              | 54% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 2 3084"
[1] "rnng bllip-sm 64924 bnc-brown 2 1701"

|======================================================                                            | 56% ~2 s remaining     [1] "rnng bllip-sm 64924 dundee 2 24605"

|=======================================================                                           | 57% ~2 s remaining     [1] "rnng bllip-sm 64924 natural-stories 2 3118"
[1] "rnng bllip-xs 4301 bnc-brown 2 1696"

|=========================================================                                         | 59% ~2 s remaining     [1] "rnng bllip-xs 4301 dundee 2 24544"

|==========================================================                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 2 3100"
[1] "rnng bllip-xs 28066 bnc-brown 2 1687"

|============================================================                                      | 62% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 2 24542"

|=============================================================                                     | 63% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 2 3101"
[1] "rnng bllip-xs 28068 bnc-brown 2 1693"

|===============================================================                                   | 65% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 2 24518"

|================================================================                                  | 66% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 2 3100"
[1] "rnng bllip-xs 51272 bnc-brown 2 1699"

|===================================================================                               | 68% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 2 24596"

|====================================================================                              | 69% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 2 3082"
[1] "vanilla bllip-lg 111 bnc-brown 2 1690"

|======================================================================                            | 72% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 2 24587"

|=======================================================================                           | 73% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 2 3121"
[1] "vanilla bllip-md 120 bnc-brown 2 1670"

|=========================================================================                         | 75% ~1 s remaining     [1] "vanilla bllip-md 120 dundee 2 24593"

|==========================================================================                        | 76% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 2 3075"
[1] "vanilla bllip-md 607 bnc-brown 2 1691"

|============================================================================                      | 78% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 2 1686"
[1] "vanilla bllip-md 922 dundee 2 24550"

|==============================================================================                    | 80% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 2 3093"
[1] "vanilla bllip-sm 111 bnc-brown 2 1686"

|================================================================================                  | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 2 24476"

|=================================================================================                 | 83% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 2 3104"
[1] "vanilla bllip-sm 120 bnc-brown 2 1703"

|===================================================================================               | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 2 24594"

|====================================================================================              | 86% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 2 3110"
[1] "vanilla bllip-sm 922 bnc-brown 2 1688"

|======================================================================================            | 88% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 2 24486"

|=======================================================================================           | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 2 3118"
[1] "vanilla bllip-xs 111 bnc-brown 2 1692"

|=========================================================================================         | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 dundee 2 24555"

|==========================================================================================        | 93% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 2 3127"
[1] "vanilla bllip-xs 120 bnc-brown 2 1689"

|============================================================================================      | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 2 24591"

|=============================================================================================     | 96% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 2 3072"
[1] "vanilla bllip-xs 922 bnc-brown 2 1708"

|===============================================================================================   | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 2 24569"

|================================================================================================  | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 2 3105"
[1] "5gram bllip-lg 1111 bnc-brown 3 1703"
[1] "5gram bllip-lg 1111 dundee 3 24540"
[1] "5gram bllip-lg 1111 natural-stories 3 3082"
[1] "5gram bllip-md 1111 bnc-brown 3 1682"
[1] "5gram bllip-md 1111 dundee 3 24557"
[1] "5gram bllip-md 1111 natural-stories 3 3077"
[1] "5gram bllip-sm 1111 bnc-brown 3 1701"
[1] "5gram bllip-sm 1111 dundee 3 24553"
[1] "5gram bllip-sm 1111 natural-stories 3 3104"
[1] "5gram bllip-xs 1111 bnc-brown 3 1685"
[1] "5gram bllip-xs 1111 dundee 3 24532"
[1] "5gram bllip-xs 1111 natural-stories 3 3105"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 3 1694"
[1] "gpt2 bllip-lg 1587139950 dundee 3 24570"
[1] "gpt2 bllip-lg 1587139950 natural-stories 3 3102"
[1] "gpt2 bllip-lg-gptbpe 1581955288 bnc-brown 3 1677"
[1] "gpt2 bllip-lg-gptbpe 1581955288 dundee 3 24544"
[1] "gpt2 bllip-lg-gptbpe 1581955288 natural-stories 3 3090"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 3 1694"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 3 24596"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 3 3123"
[1] "gpt2 bllip-md 1586986276 bnc-brown 3 1681"
[1] "gpt2 bllip-md 1586986276 dundee 3 24635"
[1] "gpt2 bllip-md 1586986276 natural-stories 3 3096"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 3 24553"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 3 3126"
[1] "gpt2 bllip-md-gptbpe 1581861474 bnc-brown 3 1692"
[1] "gpt2 bllip-md-gptbpe 1581861474 dundee 3 24511"
[1] "gpt2 bllip-md-gptbpe 1581861474 natural-stories 3 3122"
[1] "gpt2 bllip-md-gptbpe 1582126320 bnc-brown 3 1672"
[1] "gpt2 bllip-md-gptbpe 1582126320 dundee 3 24541"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 3 1721"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 3 24484"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 3 3090"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 3 1683"

|====================================                                                              | 37% ~3 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 3 24555"

|=====================================                                                             | 38% ~3 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 3 3072"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 3 1674"

|=======================================                                                           | 40% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 3 24604"

|========================================                                                          | 41% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 3 3081"
[1] "rnng bllip-lg 7245 bnc-brown 3 1700"

|==========================================                                                        | 43% ~3 s remaining     [1] "rnng bllip-lg 7245 dundee 3 24618"

|===========================================                                                       | 44% ~3 s remaining     [1] "rnng bllip-lg 7245 natural-stories 3 3127"
[1] "rnng bllip-md 3602 bnc-brown 3 1694"

|=============================================                                                     | 46% ~3 s remaining     [1] "rnng bllip-md 3602 dundee 3 24499"

|==============================================                                                    | 47% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 3 3092"
[1] "rnng bllip-md 44862 bnc-brown 3 1679"

|================================================                                                  | 49% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 3 24529"

|=================================================                                                 | 51% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 3 3083"
[1] "rnng bllip-sm 7877 bnc-brown 3 1696"

|===================================================                                               | 53% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 3 24549"

|====================================================                                              | 54% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 3 3084"
[1] "rnng bllip-sm 64924 bnc-brown 3 1671"

|======================================================                                            | 56% ~3 s remaining     [1] "rnng bllip-sm 64924 dundee 3 24599"

|=======================================================                                           | 57% ~3 s remaining     [1] "rnng bllip-sm 64924 natural-stories 3 3095"
[1] "rnng bllip-xs 4301 bnc-brown 3 1680"

|=========================================================                                         | 59% ~3 s remaining     [1] "rnng bllip-xs 4301 dundee 3 24492"

|==========================================================                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 3 3119"
[1] "rnng bllip-xs 28066 bnc-brown 3 1706"

|============================================================                                      | 62% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 3 24535"

|=============================================================                                     | 63% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 3 3114"
[1] "rnng bllip-xs 28068 bnc-brown 3 1701"

|===============================================================                                   | 65% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 3 24555"

|================================================================                                  | 66% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 3 3104"
[1] "rnng bllip-xs 51272 bnc-brown 3 1693"

|===================================================================                               | 68% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 3 24617"

|====================================================================                              | 69% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 3 3098"
[1] "vanilla bllip-lg 111 bnc-brown 3 1694"

|======================================================================                            | 72% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 3 24448"

|=======================================================================                           | 73% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 3 3089"
[1] "vanilla bllip-md 120 bnc-brown 3 1675"

|=========================================================================                         | 75% ~1 s remaining     [1] "vanilla bllip-md 120 dundee 3 24452"

|==========================================================================                        | 76% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 3 3107"
[1] "vanilla bllip-md 607 bnc-brown 3 1699"

|============================================================================                      | 78% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 3 1708"

|=============================================================================                     | 79% ~1 s remaining     [1] "vanilla bllip-md 922 dundee 3 24549"

|==============================================================================                    | 80% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 3 3128"

|===============================================================================                   | 81% ~1 s remaining     [1] "vanilla bllip-sm 111 bnc-brown 3 1674"
[1] "vanilla bllip-sm 111 dundee 3 24503"

|=================================================================================                 | 83% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 3 3108"
[1] "vanilla bllip-sm 120 bnc-brown 3 1684"

|===================================================================================               | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 3 24540"

|====================================================================================              | 86% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 3 3100"
[1] "vanilla bllip-sm 922 bnc-brown 3 1678"

|======================================================================================            | 88% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 3 24583"

|=======================================================================================           | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 3 3097"
[1] "vanilla bllip-xs 111 bnc-brown 3 1694"

|=========================================================================================         | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 dundee 3 24524"

|==========================================================================================        | 93% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 3 3112"
[1] "vanilla bllip-xs 120 bnc-brown 3 1676"

|============================================================================================      | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 3 24493"

|=============================================================================================     | 96% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 3 3122"
[1] "vanilla bllip-xs 922 bnc-brown 3 1706"

|===============================================================================================   | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 3 24581"

|================================================================================================  | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 3 3111"
[1] "5gram bllip-lg 1111 bnc-brown 3 1703"
[1] "5gram bllip-lg 1111 dundee 3 24540"
[1] "5gram bllip-lg 1111 natural-stories 3 3082"
[1] "5gram bllip-md 1111 bnc-brown 3 1682"
[1] "5gram bllip-md 1111 dundee 3 24557"
[1] "5gram bllip-md 1111 natural-stories 3 3077"
[1] "5gram bllip-sm 1111 bnc-brown 3 1701"
[1] "5gram bllip-sm 1111 dundee 3 24553"
[1] "5gram bllip-sm 1111 natural-stories 3 3104"
[1] "5gram bllip-xs 1111 bnc-brown 3 1685"
[1] "5gram bllip-xs 1111 dundee 3 24532"
[1] "5gram bllip-xs 1111 natural-stories 3 3105"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 3 1694"
[1] "gpt2 bllip-lg 1587139950 dundee 3 24570"
[1] "gpt2 bllip-lg 1587139950 natural-stories 3 3102"
[1] "gpt2 bllip-lg-gptbpe 1581955288 bnc-brown 3 1677"
[1] "gpt2 bllip-lg-gptbpe 1581955288 dundee 3 24544"
[1] "gpt2 bllip-lg-gptbpe 1581955288 natural-stories 3 3090"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 3 1694"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 3 24596"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 3 3123"
[1] "gpt2 bllip-md 1586986276 bnc-brown 3 1681"
[1] "gpt2 bllip-md 1586986276 dundee 3 24635"
[1] "gpt2 bllip-md 1586986276 natural-stories 3 3096"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 3 24553"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 3 3126"
[1] "gpt2 bllip-md-gptbpe 1581861474 bnc-brown 3 1692"
[1] "gpt2 bllip-md-gptbpe 1581861474 dundee 3 24511"
[1] "gpt2 bllip-md-gptbpe 1581861474 natural-stories 3 3122"
[1] "gpt2 bllip-md-gptbpe 1582126320 bnc-brown 3 1672"
[1] "gpt2 bllip-md-gptbpe 1582126320 dundee 3 24541"

|===============================                                                                   | 33% ~4 s remaining     [1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 3 1721"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 3 24484"

|==================================                                                                | 35% ~4 s remaining     [1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 3 3090"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 3 1683"

|====================================                                                              | 37% ~4 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 3 24555"

|=====================================                                                             | 38% ~4 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 3 3072"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 3 1674"

|=======================================                                                           | 40% ~4 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 3 24604"

|========================================                                                          | 41% ~4 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 3 3081"
[1] "rnng bllip-lg 7245 bnc-brown 3 1700"

|==========================================                                                        | 43% ~4 s remaining     [1] "rnng bllip-lg 7245 dundee 3 24618"

|===========================================                                                       | 44% ~3 s remaining     [1] "rnng bllip-lg 7245 natural-stories 3 3127"
[1] "rnng bllip-md 3602 bnc-brown 3 1694"

|=============================================                                                     | 46% ~3 s remaining     [1] "rnng bllip-md 3602 dundee 3 24499"

|==============================================                                                    | 47% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 3 3092"
[1] "rnng bllip-md 44862 bnc-brown 3 1679"

|================================================                                                  | 49% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 3 24529"

|=================================================                                                 | 51% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 3 3083"
[1] "rnng bllip-sm 7877 bnc-brown 3 1696"

|===================================================                                               | 53% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 3 24549"

|====================================================                                              | 54% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 3 3084"
[1] "rnng bllip-sm 64924 bnc-brown 3 1671"

|======================================================                                            | 56% ~3 s remaining     [1] "rnng bllip-sm 64924 dundee 3 24599"

|=======================================================                                           | 57% ~3 s remaining     [1] "rnng bllip-sm 64924 natural-stories 3 3095"
[1] "rnng bllip-xs 4301 bnc-brown 3 1680"

|=========================================================                                         | 59% ~2 s remaining     [1] "rnng bllip-xs 4301 dundee 3 24492"

|==========================================================                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 3 3119"
[1] "rnng bllip-xs 28066 bnc-brown 3 1706"

|============================================================                                      | 62% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 3 24535"

|=============================================================                                     | 63% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 3 3114"
[1] "rnng bllip-xs 28068 bnc-brown 3 1701"

|===============================================================                                   | 65% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 3 24555"

|================================================================                                  | 66% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 3 3104"
[1] "rnng bllip-xs 51272 bnc-brown 3 1693"

|===================================================================                               | 68% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 3 24617"

|====================================================================                              | 69% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 3 3098"
[1] "vanilla bllip-lg 111 bnc-brown 3 1694"

|======================================================================                            | 72% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 3 24448"

|=======================================================================                           | 73% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 3 3089"
[1] "vanilla bllip-md 120 bnc-brown 3 1675"

|=========================================================================                         | 75% ~1 s remaining     [1] "vanilla bllip-md 120 dundee 3 24452"

|==========================================================================                        | 76% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 3 3107"
[1] "vanilla bllip-md 607 bnc-brown 3 1699"

|============================================================================                      | 78% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 3 1708"
[1] "vanilla bllip-md 922 dundee 3 24549"

|==============================================================================                    | 80% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 3 3128"
[1] "vanilla bllip-sm 111 bnc-brown 3 1674"

|================================================================================                  | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 3 24503"

|=================================================================================                 | 83% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 3 3108"
[1] "vanilla bllip-sm 120 bnc-brown 3 1684"

|===================================================================================               | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 3 24540"

|====================================================================================              | 86% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 3 3100"
[1] "vanilla bllip-sm 922 bnc-brown 3 1678"

|======================================================================================            | 88% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 3 24583"

|=======================================================================================           | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 3 3097"

|========================================================================================          | 91% ~1 s remaining     [1] "vanilla bllip-xs 111 bnc-brown 3 1694"
[1] "vanilla bllip-xs 111 dundee 3 24524"

|==========================================================================================        | 93% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 3 3112"
[1] "vanilla bllip-xs 120 bnc-brown 3 1676"

|============================================================================================      | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 3 24493"

|=============================================================================================     | 96% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 3 3122"
[1] "vanilla bllip-xs 922 bnc-brown 3 1706"

|===============================================================================================   | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 3 24581"

|================================================================================================  | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 3 3111"
[1] "5gram bllip-lg 1111 bnc-brown 4 1697"
[1] "5gram bllip-lg 1111 dundee 4 24626"
[1] "5gram bllip-lg 1111 natural-stories 4 3103"
[1] "5gram bllip-md 1111 bnc-brown 4 1684"
[1] "5gram bllip-md 1111 dundee 4 24523"
[1] "5gram bllip-md 1111 natural-stories 4 3107"
[1] "5gram bllip-sm 1111 bnc-brown 4 1696"
[1] "5gram bllip-sm 1111 dundee 4 24612"
[1] "5gram bllip-sm 1111 natural-stories 4 3088"
[1] "5gram bllip-xs 1111 bnc-brown 4 1688"
[1] "5gram bllip-xs 1111 dundee 4 24565"
[1] "5gram bllip-xs 1111 natural-stories 4 3061"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 4 1692"
[1] "gpt2 bllip-lg 1587139950 dundee 4 24570"
[1] "gpt2 bllip-lg 1587139950 natural-stories 4 3087"
[1] "gpt2 bllip-lg-gptbpe 1581955288 bnc-brown 4 1709"
[1] "gpt2 bllip-lg-gptbpe 1581955288 dundee 4 24458"
[1] "gpt2 bllip-lg-gptbpe 1581955288 natural-stories 4 3104"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 4 1697"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 4 24568"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 4 3112"
[1] "gpt2 bllip-md 1586986276 bnc-brown 4 1695"
[1] "gpt2 bllip-md 1586986276 dundee 4 24567"
[1] "gpt2 bllip-md 1586986276 natural-stories 4 3123"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 4 24557"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 4 3060"
[1] "gpt2 bllip-md-gptbpe 1581861474 bnc-brown 4 1695"
[1] "gpt2 bllip-md-gptbpe 1581861474 dundee 4 24504"
[1] "gpt2 bllip-md-gptbpe 1581861474 natural-stories 4 3069"

|=============================                                                                     | 31% ~5 s remaining     [1] "gpt2 bllip-md-gptbpe 1582126320 bnc-brown 4 1688"
[1] "gpt2 bllip-md-gptbpe 1582126320 dundee 4 24564"

|===============================                                                                   | 33% ~4 s remaining     [1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 4 1682"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 4 24539"

|==================================                                                                | 35% ~4 s remaining     [1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 4 3102"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 4 1679"

|====================================                                                              | 37% ~4 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 4 24506"

|=====================================                                                             | 38% ~4 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 4 3083"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 4 1705"

|=======================================                                                           | 40% ~4 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 4 24490"

|========================================                                                          | 41% ~4 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 4 3071"
[1] "rnng bllip-lg 7245 bnc-brown 4 1692"

|==========================================                                                        | 43% ~4 s remaining     [1] "rnng bllip-lg 7245 dundee 4 24588"

|===========================================                                                       | 44% ~4 s remaining     [1] "rnng bllip-lg 7245 natural-stories 4 3081"
[1] "rnng bllip-md 3602 bnc-brown 4 1678"

|=============================================                                                     | 46% ~3 s remaining     [1] "rnng bllip-md 3602 dundee 4 24549"

|==============================================                                                    | 47% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 4 3089"
[1] "rnng bllip-md 44862 bnc-brown 4 1702"

|================================================                                                  | 49% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 4 24565"

|=================================================                                                 | 51% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 4 3040"
[1] "rnng bllip-sm 7877 bnc-brown 4 1702"

|===================================================                                               | 53% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 4 24600"

|====================================================                                              | 54% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 4 3090"
[1] "rnng bllip-sm 64924 bnc-brown 4 1710"

|======================================================                                            | 56% ~3 s remaining     [1] "rnng bllip-sm 64924 dundee 4 24508"

|=======================================================                                           | 57% ~3 s remaining     [1] "rnng bllip-sm 64924 natural-stories 4 3099"
[1] "rnng bllip-xs 4301 bnc-brown 4 1681"

|=========================================================                                         | 59% ~2 s remaining     [1] "rnng bllip-xs 4301 dundee 4 24588"

|==========================================================                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 4 3092"
[1] "rnng bllip-xs 28066 bnc-brown 4 1677"

|============================================================                                      | 62% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 4 24603"

|=============================================================                                     | 63% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 4 3106"
[1] "rnng bllip-xs 28068 bnc-brown 4 1682"

|===============================================================                                   | 65% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 4 24580"

|================================================================                                  | 66% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 4 3110"
[1] "rnng bllip-xs 51272 bnc-brown 4 1704"

|===================================================================                               | 68% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 4 24483"

|====================================================================                              | 69% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 4 3093"
[1] "vanilla bllip-lg 111 bnc-brown 4 1700"

|======================================================================                            | 72% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 4 24511"

|=======================================================================                           | 73% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 4 3100"
[1] "vanilla bllip-md 120 bnc-brown 4 1703"

|=========================================================================                         | 75% ~2 s remaining     [1] "vanilla bllip-md 120 dundee 4 24533"

|==========================================================================                        | 76% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 4 3130"
[1] "vanilla bllip-md 607 bnc-brown 4 1681"

|============================================================================                      | 78% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 4 1696"
[1] "vanilla bllip-md 922 dundee 4 24555"

|==============================================================================                    | 80% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 4 3085"
[1] "vanilla bllip-sm 111 bnc-brown 4 1684"

|================================================================================                  | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 4 24576"

|=================================================================================                 | 83% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 4 3103"
[1] "vanilla bllip-sm 120 bnc-brown 4 1695"

|===================================================================================               | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 4 24545"

|====================================================================================              | 86% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 4 3105"
[1] "vanilla bllip-sm 922 bnc-brown 4 1692"

|======================================================================================            | 88% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 4 24491"

|=======================================================================================           | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 4 3103"
[1] "vanilla bllip-xs 111 bnc-brown 4 1682"

|=========================================================================================         | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 dundee 4 24523"

|==========================================================================================        | 93% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 4 3074"
[1] "vanilla bllip-xs 120 bnc-brown 4 1693"

|============================================================================================      | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 4 24545"

|=============================================================================================     | 96% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 4 3126"
[1] "vanilla bllip-xs 922 bnc-brown 4 1680"

|===============================================================================================   | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 4 24644"

|================================================================================================  | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 4 3095"
[1] "5gram bllip-lg 1111 bnc-brown 4 1697"
[1] "5gram bllip-lg 1111 dundee 4 24626"
[1] "5gram bllip-lg 1111 natural-stories 4 3103"
[1] "5gram bllip-md 1111 bnc-brown 4 1684"
[1] "5gram bllip-md 1111 dundee 4 24523"
[1] "5gram bllip-md 1111 natural-stories 4 3107"
[1] "5gram bllip-sm 1111 bnc-brown 4 1696"
[1] "5gram bllip-sm 1111 dundee 4 24612"
[1] "5gram bllip-sm 1111 natural-stories 4 3088"
[1] "5gram bllip-xs 1111 bnc-brown 4 1688"
[1] "5gram bllip-xs 1111 dundee 4 24565"
[1] "5gram bllip-xs 1111 natural-stories 4 3061"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 4 1692"
[1] "gpt2 bllip-lg 1587139950 dundee 4 24570"
[1] "gpt2 bllip-lg 1587139950 natural-stories 4 3087"
[1] "gpt2 bllip-lg-gptbpe 1581955288 bnc-brown 4 1709"
[1] "gpt2 bllip-lg-gptbpe 1581955288 dundee 4 24458"
[1] "gpt2 bllip-lg-gptbpe 1581955288 natural-stories 4 3104"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 4 1697"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 4 24568"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 4 3112"
[1] "gpt2 bllip-md 1586986276 bnc-brown 4 1695"
[1] "gpt2 bllip-md 1586986276 dundee 4 24567"
[1] "gpt2 bllip-md 1586986276 natural-stories 4 3123"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 4 24557"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 4 3060"
[1] "gpt2 bllip-md-gptbpe 1581861474 bnc-brown 4 1695"
[1] "gpt2 bllip-md-gptbpe 1581861474 dundee 4 24504"
[1] "gpt2 bllip-md-gptbpe 1581861474 natural-stories 4 3069"
[1] "gpt2 bllip-md-gptbpe 1582126320 bnc-brown 4 1688"
[1] "gpt2 bllip-md-gptbpe 1582126320 dundee 4 24564"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 4 1682"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 4 24539"

|==================================                                                                | 35% ~4 s remaining     [1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 4 3102"

|===================================                                                               | 36% ~4 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 4 1679"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 4 24506"

|=====================================                                                             | 38% ~4 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 4 3083"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 4 1705"

|=======================================                                                           | 40% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 4 24490"

|========================================                                                          | 41% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 4 3071"
[1] "rnng bllip-lg 7245 bnc-brown 4 1692"

|==========================================                                                        | 43% ~3 s remaining     [1] "rnng bllip-lg 7245 dundee 4 24588"

|===========================================                                                       | 44% ~3 s remaining     [1] "rnng bllip-lg 7245 natural-stories 4 3081"
[1] "rnng bllip-md 3602 bnc-brown 4 1678"

|=============================================                                                     | 46% ~3 s remaining     [1] "rnng bllip-md 3602 dundee 4 24549"

|==============================================                                                    | 47% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 4 3089"
[1] "rnng bllip-md 44862 bnc-brown 4 1702"

|================================================                                                  | 49% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 4 24565"

|=================================================                                                 | 51% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 4 3040"
[1] "rnng bllip-sm 7877 bnc-brown 4 1702"

|===================================================                                               | 53% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 4 24600"

|====================================================                                              | 54% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 4 3090"
[1] "rnng bllip-sm 64924 bnc-brown 4 1710"

|======================================================                                            | 56% ~3 s remaining     [1] "rnng bllip-sm 64924 dundee 4 24508"

|=======================================================                                           | 57% ~3 s remaining     [1] "rnng bllip-sm 64924 natural-stories 4 3099"
[1] "rnng bllip-xs 4301 bnc-brown 4 1681"

|=========================================================                                         | 59% ~2 s remaining     [1] "rnng bllip-xs 4301 dundee 4 24588"

|==========================================================                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 4 3092"
[1] "rnng bllip-xs 28066 bnc-brown 4 1677"

|============================================================                                      | 62% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 4 24603"

|=============================================================                                     | 63% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 4 3106"
[1] "rnng bllip-xs 28068 bnc-brown 4 1682"

|===============================================================                                   | 65% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 4 24580"

|================================================================                                  | 66% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 4 3110"

|==================================================================                                | 67% ~2 s remaining     [1] "rnng bllip-xs 51272 bnc-brown 4 1704"
[1] "rnng bllip-xs 51272 dundee 4 24483"

|====================================================================                              | 69% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 4 3093"
[1] "vanilla bllip-lg 111 bnc-brown 4 1700"

|======================================================================                            | 72% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 4 24511"

|=======================================================================                           | 73% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 4 3100"
[1] "vanilla bllip-md 120 bnc-brown 4 1703"

|=========================================================================                         | 75% ~1 s remaining     [1] "vanilla bllip-md 120 dundee 4 24533"

|==========================================================================                        | 76% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 4 3130"
[1] "vanilla bllip-md 607 bnc-brown 4 1681"

|============================================================================                      | 78% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 4 1696"
[1] "vanilla bllip-md 922 dundee 4 24555"

|==============================================================================                    | 80% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 4 3085"
[1] "vanilla bllip-sm 111 bnc-brown 4 1684"

|================================================================================                  | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 4 24576"

|=================================================================================                 | 83% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 4 3103"
[1] "vanilla bllip-sm 120 bnc-brown 4 1695"

|===================================================================================               | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 4 24545"

|====================================================================================              | 86% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 4 3105"
[1] "vanilla bllip-sm 922 bnc-brown 4 1692"

|======================================================================================            | 88% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 4 24491"

|=======================================================================================           | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 4 3103"
[1] "vanilla bllip-xs 111 bnc-brown 4 1682"

|=========================================================================================         | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 dundee 4 24523"

|==========================================================================================        | 93% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 4 3074"
[1] "vanilla bllip-xs 120 bnc-brown 4 1693"

|============================================================================================      | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 4 24545"

|=============================================================================================     | 96% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 4 3126"

|==============================================================================================    | 97% ~0 s remaining     [1] "vanilla bllip-xs 922 bnc-brown 4 1680"
[1] "vanilla bllip-xs 922 dundee 4 24644"

|================================================================================================  | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 4 3095"
[1] "5gram bllip-lg 1111 bnc-brown 5 1695"
[1] "5gram bllip-lg 1111 dundee 5 24529"
[1] "5gram bllip-lg 1111 natural-stories 5 3120"
[1] "5gram bllip-md 1111 bnc-brown 5 1672"
[1] "5gram bllip-md 1111 dundee 5 24510"
[1] "5gram bllip-md 1111 natural-stories 5 3106"
[1] "5gram bllip-sm 1111 bnc-brown 5 1704"
[1] "5gram bllip-sm 1111 dundee 5 24547"
[1] "5gram bllip-sm 1111 natural-stories 5 3083"
[1] "5gram bllip-xs 1111 bnc-brown 5 1694"
[1] "5gram bllip-xs 1111 dundee 5 24470"
[1] "5gram bllip-xs 1111 natural-stories 5 3122"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 5 1706"
[1] "gpt2 bllip-lg 1587139950 dundee 5 24535"
[1] "gpt2 bllip-lg 1587139950 natural-stories 5 3095"
[1] "gpt2 bllip-lg-gptbpe 1581955288 bnc-brown 5 1680"
[1] "gpt2 bllip-lg-gptbpe 1581955288 dundee 5 24557"
[1] "gpt2 bllip-lg-gptbpe 1581955288 natural-stories 5 3079"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 5 1702"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 5 24552"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 5 3073"
[1] "gpt2 bllip-md 1586986276 bnc-brown 5 1695"
[1] "gpt2 bllip-md 1586986276 dundee 5 24522"
[1] "gpt2 bllip-md 1586986276 natural-stories 5 3100"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 5 24573"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 5 3112"
[1] "gpt2 bllip-md-gptbpe 1581861474 bnc-brown 5 1707"
[1] "gpt2 bllip-md-gptbpe 1581861474 dundee 5 24601"
[1] "gpt2 bllip-md-gptbpe 1581861474 natural-stories 5 3088"
[1] "gpt2 bllip-md-gptbpe 1582126320 bnc-brown 5 1708"
[1] "gpt2 bllip-md-gptbpe 1582126320 dundee 5 24519"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 5 1715"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 5 24514"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 5 3077"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 5 1693"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 5 24687"

|=====================================                                                             | 38% ~4 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 5 3102"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 5 1698"

|=======================================                                                           | 40% ~4 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 5 24573"

|========================================                                                          | 41% ~4 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 5 3122"
[1] "rnng bllip-lg 7245 bnc-brown 5 1701"

|==========================================                                                        | 43% ~4 s remaining     [1] "rnng bllip-lg 7245 dundee 5 24474"

|===========================================                                                       | 44% ~4 s remaining     [1] "rnng bllip-lg 7245 natural-stories 5 3118"
[1] "rnng bllip-md 3602 bnc-brown 5 1679"

|=============================================                                                     | 46% ~3 s remaining     [1] "rnng bllip-md 3602 dundee 5 24426"

|==============================================                                                    | 47% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 5 3101"
[1] "rnng bllip-md 44862 bnc-brown 5 1692"

|================================================                                                  | 49% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 5 24537"

|=================================================                                                 | 51% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 5 3116"
[1] "rnng bllip-sm 7877 bnc-brown 5 1683"

|===================================================                                               | 53% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 5 24577"

|====================================================                                              | 54% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 5 3106"
[1] "rnng bllip-sm 64924 bnc-brown 5 1709"

|======================================================                                            | 56% ~3 s remaining     [1] "rnng bllip-sm 64924 dundee 5 24504"

|=======================================================                                           | 57% ~3 s remaining     [1] "rnng bllip-sm 64924 natural-stories 5 3084"
[1] "rnng bllip-xs 4301 bnc-brown 5 1694"

|=========================================================                                         | 59% ~2 s remaining     [1] "rnng bllip-xs 4301 dundee 5 24568"

|==========================================================                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 5 3074"
[1] "rnng bllip-xs 28066 bnc-brown 5 1705"

|============================================================                                      | 62% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 5 24574"

|=============================================================                                     | 63% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 5 3092"
[1] "rnng bllip-xs 28068 bnc-brown 5 1687"

|===============================================================                                   | 65% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 5 24592"

|================================================================                                  | 66% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 5 3084"
[1] "rnng bllip-xs 51272 bnc-brown 5 1702"

|===================================================================                               | 68% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 5 24552"

|====================================================================                              | 69% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 5 3060"
[1] "vanilla bllip-lg 111 bnc-brown 5 1687"

|======================================================================                            | 72% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 5 24485"

|=======================================================================                           | 73% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 5 3103"
[1] "vanilla bllip-md 120 bnc-brown 5 1703"

|=========================================================================                         | 75% ~1 s remaining     [1] "vanilla bllip-md 120 dundee 5 24586"

|==========================================================================                        | 76% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 5 3107"
[1] "vanilla bllip-md 607 bnc-brown 5 1688"

|============================================================================                      | 78% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 5 1698"
[1] "vanilla bllip-md 922 dundee 5 24554"

|==============================================================================                    | 80% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 5 3077"
[1] "vanilla bllip-sm 111 bnc-brown 5 1704"

|================================================================================                  | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 5 24546"

|=================================================================================                 | 83% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 5 3101"
[1] "vanilla bllip-sm 120 bnc-brown 5 1705"

|===================================================================================               | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 5 24507"

|====================================================================================              | 86% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 5 3089"

|=====================================================================================             | 87% ~1 s remaining     [1] "vanilla bllip-sm 922 bnc-brown 5 1706"
[1] "vanilla bllip-sm 922 dundee 5 24588"

|=======================================================================================           | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 5 3096"
[1] "vanilla bllip-xs 111 bnc-brown 5 1687"

|=========================================================================================         | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 dundee 5 24569"

|==========================================================================================        | 93% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 5 3128"
[1] "vanilla bllip-xs 120 bnc-brown 5 1712"

|============================================================================================      | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 5 24501"

|=============================================================================================     | 96% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 5 3109"
[1] "vanilla bllip-xs 922 bnc-brown 5 1691"

|===============================================================================================   | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 5 24515"

|================================================================================================  | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 5 3098"
[1] "5gram bllip-lg 1111 bnc-brown 5 1695"
[1] "5gram bllip-lg 1111 dundee 5 24529"
[1] "5gram bllip-lg 1111 natural-stories 5 3120"
[1] "5gram bllip-md 1111 bnc-brown 5 1672"
[1] "5gram bllip-md 1111 dundee 5 24510"
[1] "5gram bllip-md 1111 natural-stories 5 3106"
[1] "5gram bllip-sm 1111 bnc-brown 5 1704"
[1] "5gram bllip-sm 1111 dundee 5 24547"
[1] "5gram bllip-sm 1111 natural-stories 5 3083"
[1] "5gram bllip-xs 1111 bnc-brown 5 1694"
[1] "5gram bllip-xs 1111 dundee 5 24470"
[1] "5gram bllip-xs 1111 natural-stories 5 3122"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 5 1706"
[1] "gpt2 bllip-lg 1587139950 dundee 5 24535"
[1] "gpt2 bllip-lg 1587139950 natural-stories 5 3095"
[1] "gpt2 bllip-lg-gptbpe 1581955288 bnc-brown 5 1680"
[1] "gpt2 bllip-lg-gptbpe 1581955288 dundee 5 24557"
[1] "gpt2 bllip-lg-gptbpe 1581955288 natural-stories 5 3079"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 5 1702"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 5 24552"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 5 3073"
[1] "gpt2 bllip-md 1586986276 bnc-brown 5 1695"
[1] "gpt2 bllip-md 1586986276 dundee 5 24522"
[1] "gpt2 bllip-md 1586986276 natural-stories 5 3100"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 5 24573"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 5 3112"
[1] "gpt2 bllip-md-gptbpe 1581861474 bnc-brown 5 1707"
[1] "gpt2 bllip-md-gptbpe 1581861474 dundee 5 24601"
[1] "gpt2 bllip-md-gptbpe 1581861474 natural-stories 5 3088"
[1] "gpt2 bllip-md-gptbpe 1582126320 bnc-brown 5 1708"
[1] "gpt2 bllip-md-gptbpe 1582126320 dundee 5 24519"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 5 1715"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 5 24514"

|==================================                                                                | 35% ~4 s remaining     [1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 5 3077"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 5 1693"

|====================================                                                              | 37% ~4 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 5 24687"

|=====================================                                                             | 38% ~4 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 5 3102"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 5 1698"

|=======================================                                                           | 40% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 5 24573"

|========================================                                                          | 41% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 5 3122"
[1] "rnng bllip-lg 7245 bnc-brown 5 1701"

|==========================================                                                        | 43% ~3 s remaining     [1] "rnng bllip-lg 7245 dundee 5 24474"

|===========================================                                                       | 44% ~3 s remaining     [1] "rnng bllip-lg 7245 natural-stories 5 3118"

|============================================                                                      | 45% ~3 s remaining     [1] "rnng bllip-md 3602 bnc-brown 5 1679"
[1] "rnng bllip-md 3602 dundee 5 24426"

|==============================================                                                    | 47% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 5 3101"
[1] "rnng bllip-md 44862 bnc-brown 5 1692"

|================================================                                                  | 49% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 5 24537"

|=================================================                                                 | 51% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 5 3116"
[1] "rnng bllip-sm 7877 bnc-brown 5 1683"

|===================================================                                               | 53% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 5 24577"

|====================================================                                              | 54% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 5 3106"
[1] "rnng bllip-sm 64924 bnc-brown 5 1709"

|======================================================                                            | 56% ~3 s remaining     [1] "rnng bllip-sm 64924 dundee 5 24504"

|=======================================================                                           | 57% ~3 s remaining     [1] "rnng bllip-sm 64924 natural-stories 5 3084"
[1] "rnng bllip-xs 4301 bnc-brown 5 1694"

|=========================================================                                         | 59% ~2 s remaining     [1] "rnng bllip-xs 4301 dundee 5 24568"

|==========================================================                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 5 3074"
[1] "rnng bllip-xs 28066 bnc-brown 5 1705"

|============================================================                                      | 62% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 5 24574"

|=============================================================                                     | 63% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 5 3092"

|==============================================================                                    | 64% ~2 s remaining     [1] "rnng bllip-xs 28068 bnc-brown 5 1687"
[1] "rnng bllip-xs 28068 dundee 5 24592"

|================================================================                                  | 66% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 5 3084"
[1] "rnng bllip-xs 51272 bnc-brown 5 1702"

|===================================================================                               | 68% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 5 24552"

|====================================================================                              | 69% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 5 3060"
[1] "vanilla bllip-lg 111 bnc-brown 5 1687"

|======================================================================                            | 72% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 5 24485"

|=======================================================================                           | 73% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 5 3103"
[1] "vanilla bllip-md 120 bnc-brown 5 1703"

|=========================================================================                         | 75% ~1 s remaining     [1] "vanilla bllip-md 120 dundee 5 24586"

|==========================================================================                        | 76% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 5 3107"
[1] "vanilla bllip-md 607 bnc-brown 5 1688"

|============================================================================                      | 78% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 5 1698"
[1] "vanilla bllip-md 922 dundee 5 24554"

|==============================================================================                    | 80% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 5 3077"

|===============================================================================                   | 81% ~1 s remaining     [1] "vanilla bllip-sm 111 bnc-brown 5 1704"

|================================================================================                  | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 5 24546"

|=================================================================================                 | 83% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 5 3101"
[1] "vanilla bllip-sm 120 bnc-brown 5 1705"

|===================================================================================               | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 5 24507"

|====================================================================================              | 86% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 5 3089"

|=====================================================================================             | 87% ~1 s remaining     [1] "vanilla bllip-sm 922 bnc-brown 5 1706"
[1] "vanilla bllip-sm 922 dundee 5 24588"

|=======================================================================================           | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 5 3096"

|========================================================================================          | 91% ~1 s remaining     [1] "vanilla bllip-xs 111 bnc-brown 5 1687"
[1] "vanilla bllip-xs 111 dundee 5 24569"

|==========================================================================================        | 93% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 5 3128"
[1] "vanilla bllip-xs 120 bnc-brown 5 1712"

|============================================================================================      | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 5 24501"

|=============================================================================================     | 96% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 5 3109"
[1] "vanilla bllip-xs 922 bnc-brown 5 1691"

|===============================================================================================   | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 5 24515"

|================================================================================================  | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 5 3098"
[1] "5gram bllip-lg 1111 bnc-brown 6 1683"
[1] "5gram bllip-lg 1111 dundee 6 24583"
[1] "5gram bllip-lg 1111 natural-stories 6 3082"
[1] "5gram bllip-md 1111 bnc-brown 6 1724"
[1] "5gram bllip-md 1111 dundee 6 24575"
[1] "5gram bllip-md 1111 natural-stories 6 3087"
[1] "5gram bllip-sm 1111 bnc-brown 6 1712"
[1] "5gram bllip-sm 1111 dundee 6 24567"
[1] "5gram bllip-sm 1111 natural-stories 6 3127"
[1] "5gram bllip-xs 1111 bnc-brown 6 1694"
[1] "5gram bllip-xs 1111 dundee 6 24573"
[1] "5gram bllip-xs 1111 natural-stories 6 3092"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 6 1691"
[1] "gpt2 bllip-lg 1587139950 dundee 6 24572"
[1] "gpt2 bllip-lg 1587139950 natural-stories 6 3110"
[1] "gpt2 bllip-lg-gptbpe 1581955288 bnc-brown 6 1700"
[1] "gpt2 bllip-lg-gptbpe 1581955288 dundee 6 24584"
[1] "gpt2 bllip-lg-gptbpe 1581955288 natural-stories 6 3069"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 6 1675"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 6 24565"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 6 3092"
[1] "gpt2 bllip-md 1586986276 bnc-brown 6 1697"
[1] "gpt2 bllip-md 1586986276 dundee 6 24572"
[1] "gpt2 bllip-md 1586986276 natural-stories 6 3066"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 6 24608"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 6 3080"
[1] "gpt2 bllip-md-gptbpe 1581861474 bnc-brown 6 1658"
[1] "gpt2 bllip-md-gptbpe 1581861474 dundee 6 24613"
[1] "gpt2 bllip-md-gptbpe 1581861474 natural-stories 6 3090"
[1] "gpt2 bllip-md-gptbpe 1582126320 bnc-brown 6 1695"
[1] "gpt2 bllip-md-gptbpe 1582126320 dundee 6 24467"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 6 1705"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 6 24597"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 6 3097"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 6 1688"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 6 24547"

|=====================================                                                             | 38% ~3 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 6 3085"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 6 1660"

|=======================================                                                           | 40% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 6 24619"

|========================================                                                          | 41% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 6 3111"
[1] "rnng bllip-lg 7245 bnc-brown 6 1693"

|==========================================                                                        | 43% ~3 s remaining     [1] "rnng bllip-lg 7245 dundee 6 24494"

|===========================================                                                       | 44% ~4 s remaining     [1] "rnng bllip-lg 7245 natural-stories 6 3098"
[1] "rnng bllip-md 3602 bnc-brown 6 1702"

|=============================================                                                     | 46% ~3 s remaining     [1] "rnng bllip-md 3602 dundee 6 24484"

|==============================================                                                    | 47% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 6 3099"

|===============================================                                                   | 48% ~3 s remaining     [1] "rnng bllip-md 44862 bnc-brown 6 1683"
[1] "rnng bllip-md 44862 dundee 6 24539"

|=================================================                                                 | 51% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 6 3103"
[1] "rnng bllip-sm 7877 bnc-brown 6 1696"

|===================================================                                               | 53% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 6 24483"

|====================================================                                              | 54% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 6 3092"
[1] "rnng bllip-sm 64924 bnc-brown 6 1698"

|======================================================                                            | 56% ~3 s remaining     [1] "rnng bllip-sm 64924 dundee 6 24558"

|=======================================================                                           | 57% ~3 s remaining     [1] "rnng bllip-sm 64924 natural-stories 6 3100"
[1] "rnng bllip-xs 4301 bnc-brown 6 1693"

|=========================================================                                         | 59% ~3 s remaining     [1] "rnng bllip-xs 4301 dundee 6 24554"

|==========================================================                                        | 60% ~3 s remaining     [1] "rnng bllip-xs 4301 natural-stories 6 3097"
[1] "rnng bllip-xs 28066 bnc-brown 6 1693"

|============================================================                                      | 62% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 6 24539"

|=============================================================                                     | 63% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 6 3110"
[1] "rnng bllip-xs 28068 bnc-brown 6 1679"

|===============================================================                                   | 65% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 6 24635"

|================================================================                                  | 66% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 6 3076"
[1] "rnng bllip-xs 51272 bnc-brown 6 1679"

|===================================================================                               | 68% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 6 24539"

|====================================================================                              | 69% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 6 3103"
[1] "vanilla bllip-lg 111 bnc-brown 6 1703"

|======================================================================                            | 72% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 6 24581"

|=======================================================================                           | 73% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 6 3080"
[1] "vanilla bllip-md 120 bnc-brown 6 1696"

|=========================================================================                         | 75% ~2 s remaining     [1] "vanilla bllip-md 120 dundee 6 24528"

|==========================================================================                        | 76% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 6 3082"

|===========================================================================                       | 77% ~1 s remaining     [1] "vanilla bllip-md 607 bnc-brown 6 1686"
[1] "vanilla bllip-md 922 bnc-brown 6 1685"

|=============================================================================                     | 79% ~1 s remaining     [1] "vanilla bllip-md 922 dundee 6 24543"

|==============================================================================                    | 80% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 6 3095"
[1] "vanilla bllip-sm 111 bnc-brown 6 1711"

|================================================================================                  | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 6 24431"

|=================================================================================                 | 83% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 6 3107"
[1] "vanilla bllip-sm 120 bnc-brown 6 1690"

|===================================================================================               | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 6 24477"

|====================================================================================              | 86% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 6 3100"
[1] "vanilla bllip-sm 922 bnc-brown 6 1676"

|======================================================================================            | 88% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 6 24575"

|=======================================================================================           | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 6 3082"
[1] "vanilla bllip-xs 111 bnc-brown 6 1697"

|=========================================================================================         | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 dundee 6 24572"

|==========================================================================================        | 93% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 6 3099"
[1] "vanilla bllip-xs 120 bnc-brown 6 1712"

|============================================================================================      | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 6 24608"

|=============================================================================================     | 96% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 6 3092"

|==============================================================================================    | 97% ~0 s remaining     [1] "vanilla bllip-xs 922 bnc-brown 6 1674"
[1] "vanilla bllip-xs 922 dundee 6 24450"

|================================================================================================  | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 6 3105"

|==================================================================================================|100% ~0 s remaining     [1] "5gram bllip-lg 1111 bnc-brown 6 1683"
[1] "5gram bllip-lg 1111 dundee 6 24583"
[1] "5gram bllip-lg 1111 natural-stories 6 3082"
[1] "5gram bllip-md 1111 bnc-brown 6 1724"
[1] "5gram bllip-md 1111 dundee 6 24575"
[1] "5gram bllip-md 1111 natural-stories 6 3087"
[1] "5gram bllip-sm 1111 bnc-brown 6 1712"
[1] "5gram bllip-sm 1111 dundee 6 24567"
[1] "5gram bllip-sm 1111 natural-stories 6 3127"
[1] "5gram bllip-xs 1111 bnc-brown 6 1694"
[1] "5gram bllip-xs 1111 dundee 6 24573"
[1] "5gram bllip-xs 1111 natural-stories 6 3092"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 6 1691"
[1] "gpt2 bllip-lg 1587139950 dundee 6 24572"
[1] "gpt2 bllip-lg 1587139950 natural-stories 6 3110"
[1] "gpt2 bllip-lg-gptbpe 1581955288 bnc-brown 6 1700"
[1] "gpt2 bllip-lg-gptbpe 1581955288 dundee 6 24584"
[1] "gpt2 bllip-lg-gptbpe 1581955288 natural-stories 6 3069"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 6 1675"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 6 24565"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 6 3092"
[1] "gpt2 bllip-md 1586986276 bnc-brown 6 1697"
[1] "gpt2 bllip-md 1586986276 dundee 6 24572"
[1] "gpt2 bllip-md 1586986276 natural-stories 6 3066"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 6 24608"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 6 3080"
[1] "gpt2 bllip-md-gptbpe 1581861474 bnc-brown 6 1658"
[1] "gpt2 bllip-md-gptbpe 1581861474 dundee 6 24613"
[1] "gpt2 bllip-md-gptbpe 1581861474 natural-stories 6 3090"
[1] "gpt2 bllip-md-gptbpe 1582126320 bnc-brown 6 1695"
[1] "gpt2 bllip-md-gptbpe 1582126320 dundee 6 24467"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 6 1705"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 6 24597"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 6 3097"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 6 1688"

|====================================                                                              | 37% ~3 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 6 24547"

|=====================================                                                             | 38% ~4 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 6 3085"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 6 1660"

|=======================================                                                           | 40% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 6 24619"

|========================================                                                          | 41% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 6 3111"
[1] "rnng bllip-lg 7245 bnc-brown 6 1693"

|==========================================                                                        | 43% ~3 s remaining     [1] "rnng bllip-lg 7245 dundee 6 24494"

|===========================================                                                       | 44% ~3 s remaining     [1] "rnng bllip-lg 7245 natural-stories 6 3098"
[1] "rnng bllip-md 3602 bnc-brown 6 1702"

|=============================================                                                     | 46% ~3 s remaining     [1] "rnng bllip-md 3602 dundee 6 24484"

|==============================================                                                    | 47% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 6 3099"
[1] "rnng bllip-md 44862 bnc-brown 6 1683"

|================================================                                                  | 49% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 6 24539"

|=================================================                                                 | 51% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 6 3103"
[1] "rnng bllip-sm 7877 bnc-brown 6 1696"

|===================================================                                               | 53% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 6 24483"

|====================================================                                              | 54% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 6 3092"
[1] "rnng bllip-sm 64924 bnc-brown 6 1698"

|======================================================                                            | 56% ~3 s remaining     [1] "rnng bllip-sm 64924 dundee 6 24558"

|=======================================================                                           | 57% ~2 s remaining     [1] "rnng bllip-sm 64924 natural-stories 6 3100"

|========================================================                                          | 58% ~2 s remaining     [1] "rnng bllip-xs 4301 bnc-brown 6 1693"
[1] "rnng bllip-xs 4301 dundee 6 24554"

|==========================================================                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 6 3097"
[1] "rnng bllip-xs 28066 bnc-brown 6 1693"

|============================================================                                      | 62% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 6 24539"

|=============================================================                                     | 63% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 6 3110"
[1] "rnng bllip-xs 28068 bnc-brown 6 1679"

|===============================================================                                   | 65% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 6 24635"

|================================================================                                  | 66% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 6 3076"
[1] "rnng bllip-xs 51272 bnc-brown 6 1679"

|===================================================================                               | 68% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 6 24539"

|====================================================================                              | 69% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 6 3103"
[1] "vanilla bllip-lg 111 bnc-brown 6 1703"

|======================================================================                            | 72% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 6 24581"

|=======================================================================                           | 73% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 6 3080"
[1] "vanilla bllip-md 120 bnc-brown 6 1696"

|=========================================================================                         | 75% ~1 s remaining     [1] "vanilla bllip-md 120 dundee 6 24528"

|==========================================================================                        | 76% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 6 3082"
[1] "vanilla bllip-md 607 bnc-brown 6 1686"

|============================================================================                      | 78% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 6 1685"
[1] "vanilla bllip-md 922 dundee 6 24543"

|==============================================================================                    | 80% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 6 3095"
[1] "vanilla bllip-sm 111 bnc-brown 6 1711"

|================================================================================                  | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 6 24431"

|=================================================================================                 | 83% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 6 3107"
[1] "vanilla bllip-sm 120 bnc-brown 6 1690"

|===================================================================================               | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 6 24477"

|====================================================================================              | 86% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 6 3100"
[1] "vanilla bllip-sm 922 bnc-brown 6 1676"

|======================================================================================            | 88% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 6 24575"

|=======================================================================================           | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 6 3082"
[1] "vanilla bllip-xs 111 bnc-brown 6 1697"

|=========================================================================================         | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 dundee 6 24572"

|==========================================================================================        | 93% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 6 3099"
[1] "vanilla bllip-xs 120 bnc-brown 6 1712"

|============================================================================================      | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 6 24608"

|=============================================================================================     | 96% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 6 3092"
[1] "vanilla bllip-xs 922 bnc-brown 6 1674"

|===============================================================================================   | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 6 24450"

|================================================================================================  | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 6 3105"
[1] "5gram bllip-lg 1111 bnc-brown 7 1690"
[1] "5gram bllip-lg 1111 dundee 7 24566"
[1] "5gram bllip-lg 1111 natural-stories 7 3101"
[1] "5gram bllip-md 1111 bnc-brown 7 1677"
[1] "5gram bllip-md 1111 dundee 7 24554"
[1] "5gram bllip-md 1111 natural-stories 7 3105"
[1] "5gram bllip-sm 1111 bnc-brown 7 1688"
[1] "5gram bllip-sm 1111 dundee 7 24617"
[1] "5gram bllip-sm 1111 natural-stories 7 3102"
[1] "5gram bllip-xs 1111 bnc-brown 7 1727"
[1] "5gram bllip-xs 1111 dundee 7 24592"
[1] "5gram bllip-xs 1111 natural-stories 7 3099"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 7 1682"
[1] "gpt2 bllip-lg 1587139950 dundee 7 24598"
[1] "gpt2 bllip-lg 1587139950 natural-stories 7 3130"
[1] "gpt2 bllip-lg-gptbpe 1581955288 bnc-brown 7 1703"
[1] "gpt2 bllip-lg-gptbpe 1581955288 dundee 7 24588"
[1] "gpt2 bllip-lg-gptbpe 1581955288 natural-stories 7 3098"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 7 1695"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 7 24439"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 7 3093"
[1] "gpt2 bllip-md 1586986276 bnc-brown 7 1695"
[1] "gpt2 bllip-md 1586986276 dundee 7 24447"
[1] "gpt2 bllip-md 1586986276 natural-stories 7 3096"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 7 24467"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 7 3084"
[1] "gpt2 bllip-md-gptbpe 1581861474 bnc-brown 7 1703"
[1] "gpt2 bllip-md-gptbpe 1581861474 dundee 7 24614"
[1] "gpt2 bllip-md-gptbpe 1581861474 natural-stories 7 3078"
[1] "gpt2 bllip-md-gptbpe 1582126320 bnc-brown 7 1689"
[1] "gpt2 bllip-md-gptbpe 1582126320 dundee 7 24552"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 7 1710"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 7 24533"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 7 3101"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 7 1698"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 7 24523"
[1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 7 3128"

|======================================                                                            | 39% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 7 1710"
[1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 7 24563"

|========================================                                                          | 41% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 7 3111"
[1] "rnng bllip-lg 7245 bnc-brown 7 1698"

|==========================================                                                        | 43% ~3 s remaining     [1] "rnng bllip-lg 7245 dundee 7 24556"

|===========================================                                                       | 44% ~3 s remaining     [1] "rnng bllip-lg 7245 natural-stories 7 3127"
[1] "rnng bllip-md 3602 bnc-brown 7 1690"

|=============================================                                                     | 46% ~3 s remaining     [1] "rnng bllip-md 3602 dundee 7 24602"

|==============================================                                                    | 47% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 7 3088"
[1] "rnng bllip-md 44862 bnc-brown 7 1708"

|================================================                                                  | 49% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 7 24570"

|=================================================                                                 | 51% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 7 3095"
[1] "rnng bllip-sm 7877 bnc-brown 7 1696"

|===================================================                                               | 53% ~2 s remaining     [1] "rnng bllip-sm 7877 dundee 7 24532"

|====================================================                                              | 54% ~2 s remaining     [1] "rnng bllip-sm 7877 natural-stories 7 3080"
[1] "rnng bllip-sm 64924 bnc-brown 7 1673"

|======================================================                                            | 56% ~2 s remaining     [1] "rnng bllip-sm 64924 dundee 7 24581"

|=======================================================                                           | 57% ~2 s remaining     [1] "rnng bllip-sm 64924 natural-stories 7 3106"
[1] "rnng bllip-xs 4301 bnc-brown 7 1696"

|=========================================================                                         | 59% ~2 s remaining     [1] "rnng bllip-xs 4301 dundee 7 24594"

|==========================================================                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 7 3064"
[1] "rnng bllip-xs 28066 bnc-brown 7 1681"

|============================================================                                      | 62% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 7 24485"

|=============================================================                                     | 63% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 7 3052"
[1] "rnng bllip-xs 28068 bnc-brown 7 1686"

|===============================================================                                   | 65% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 7 24458"

|================================================================                                  | 66% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 7 3117"
[1] "rnng bllip-xs 51272 bnc-brown 7 1664"

|===================================================================                               | 68% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 7 24526"

|====================================================================                              | 69% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 7 3118"
[1] "vanilla bllip-lg 111 bnc-brown 7 1690"

|======================================================================                            | 72% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 7 24520"

|=======================================================================                           | 73% ~1 s remaining     [1] "vanilla bllip-lg 111 natural-stories 7 3103"
[1] "vanilla bllip-md 120 bnc-brown 7 1663"

|=========================================================================                         | 75% ~1 s remaining     [1] "vanilla bllip-md 120 dundee 7 24522"

|==========================================================================                        | 76% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 7 3079"
[1] "vanilla bllip-md 607 bnc-brown 7 1701"

|============================================================================                      | 78% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 7 1700"
[1] "vanilla bllip-md 922 dundee 7 24500"

|==============================================================================                    | 80% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 7 3063"
[1] "vanilla bllip-sm 111 bnc-brown 7 1710"

|================================================================================                  | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 7 24646"

|=================================================================================                 | 83% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 7 3079"
[1] "vanilla bllip-sm 120 bnc-brown 7 1676"

|===================================================================================               | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 7 24583"

|====================================================================================              | 86% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 7 3104"
[1] "vanilla bllip-sm 922 bnc-brown 7 1702"

|======================================================================================            | 88% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 7 24515"

|=======================================================================================           | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 7 3102"
[1] "vanilla bllip-xs 111 bnc-brown 7 1698"

|=========================================================================================         | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 dundee 7 24531"

|==========================================================================================        | 93% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 7 3100"
[1] "vanilla bllip-xs 120 bnc-brown 7 1664"

|============================================================================================      | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 7 24499"

|=============================================================================================     | 96% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 7 3112"
[1] "vanilla bllip-xs 922 bnc-brown 7 1672"

|===============================================================================================   | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 7 24639"

|================================================================================================  | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 7 3106"
[1] "5gram bllip-lg 1111 bnc-brown 7 1690"
[1] "5gram bllip-lg 1111 dundee 7 24566"
[1] "5gram bllip-lg 1111 natural-stories 7 3101"
[1] "5gram bllip-md 1111 bnc-brown 7 1677"
[1] "5gram bllip-md 1111 dundee 7 24554"
[1] "5gram bllip-md 1111 natural-stories 7 3105"
[1] "5gram bllip-sm 1111 bnc-brown 7 1688"
[1] "5gram bllip-sm 1111 dundee 7 24617"
[1] "5gram bllip-sm 1111 natural-stories 7 3102"
[1] "5gram bllip-xs 1111 bnc-brown 7 1727"
[1] "5gram bllip-xs 1111 dundee 7 24592"
[1] "5gram bllip-xs 1111 natural-stories 7 3099"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 7 1682"
[1] "gpt2 bllip-lg 1587139950 dundee 7 24598"
[1] "gpt2 bllip-lg 1587139950 natural-stories 7 3130"
[1] "gpt2 bllip-lg-gptbpe 1581955288 bnc-brown 7 1703"
[1] "gpt2 bllip-lg-gptbpe 1581955288 dundee 7 24588"
[1] "gpt2 bllip-lg-gptbpe 1581955288 natural-stories 7 3098"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 7 1695"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 7 24439"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 7 3093"
[1] "gpt2 bllip-md 1586986276 bnc-brown 7 1695"
[1] "gpt2 bllip-md 1586986276 dundee 7 24447"
[1] "gpt2 bllip-md 1586986276 natural-stories 7 3096"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 7 24467"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 7 3084"
[1] "gpt2 bllip-md-gptbpe 1581861474 bnc-brown 7 1703"
[1] "gpt2 bllip-md-gptbpe 1581861474 dundee 7 24614"
[1] "gpt2 bllip-md-gptbpe 1581861474 natural-stories 7 3078"
[1] "gpt2 bllip-md-gptbpe 1582126320 bnc-brown 7 1689"
[1] "gpt2 bllip-md-gptbpe 1582126320 dundee 7 24552"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 7 1710"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 7 24533"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 7 3101"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 7 1698"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 7 24523"

|=====================================                                                             | 38% ~3 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 7 3128"

|======================================                                                            | 39% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 7 1710"
[1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 7 24563"

|========================================                                                          | 41% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 7 3111"
[1] "rnng bllip-lg 7245 bnc-brown 7 1698"

|==========================================                                                        | 43% ~3 s remaining     [1] "rnng bllip-lg 7245 dundee 7 24556"

|===========================================                                                       | 44% ~3 s remaining     [1] "rnng bllip-lg 7245 natural-stories 7 3127"
[1] "rnng bllip-md 3602 bnc-brown 7 1690"

|=============================================                                                     | 46% ~3 s remaining     [1] "rnng bllip-md 3602 dundee 7 24602"

|==============================================                                                    | 47% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 7 3088"
[1] "rnng bllip-md 44862 bnc-brown 7 1708"

|================================================                                                  | 49% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 7 24570"

|=================================================                                                 | 51% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 7 3095"
[1] "rnng bllip-sm 7877 bnc-brown 7 1696"

|===================================================                                               | 53% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 7 24532"

|====================================================                                              | 54% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 7 3080"
[1] "rnng bllip-sm 64924 bnc-brown 7 1673"

|======================================================                                            | 56% ~2 s remaining     [1] "rnng bllip-sm 64924 dundee 7 24581"

|=======================================================                                           | 57% ~2 s remaining     [1] "rnng bllip-sm 64924 natural-stories 7 3106"
[1] "rnng bllip-xs 4301 bnc-brown 7 1696"

|=========================================================                                         | 59% ~2 s remaining     [1] "rnng bllip-xs 4301 dundee 7 24594"

|==========================================================                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 7 3064"

|===========================================================                                       | 61% ~2 s remaining     [1] "rnng bllip-xs 28066 bnc-brown 7 1681"
[1] "rnng bllip-xs 28066 dundee 7 24485"

|=============================================================                                     | 63% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 7 3052"
[1] "rnng bllip-xs 28068 bnc-brown 7 1686"

|===============================================================                                   | 65% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 7 24458"

|================================================================                                  | 66% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 7 3117"

|==================================================================                                | 67% ~2 s remaining     [1] "rnng bllip-xs 51272 bnc-brown 7 1664"
[1] "rnng bllip-xs 51272 dundee 7 24526"

|====================================================================                              | 69% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 7 3118"
[1] "vanilla bllip-lg 111 bnc-brown 7 1690"

|======================================================================                            | 72% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 7 24520"

|=======================================================================                           | 73% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 7 3103"
[1] "vanilla bllip-md 120 bnc-brown 7 1663"

|=========================================================================                         | 75% ~1 s remaining     [1] "vanilla bllip-md 120 dundee 7 24522"

|==========================================================================                        | 76% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 7 3079"
[1] "vanilla bllip-md 607 bnc-brown 7 1701"

|============================================================================                      | 78% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 7 1700"
[1] "vanilla bllip-md 922 dundee 7 24500"

|==============================================================================                    | 80% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 7 3063"
[1] "vanilla bllip-sm 111 bnc-brown 7 1710"

|================================================================================                  | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 7 24646"

|=================================================================================                 | 83% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 7 3079"
[1] "vanilla bllip-sm 120 bnc-brown 7 1676"

|===================================================================================               | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 7 24583"

|====================================================================================              | 86% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 7 3104"
[1] "vanilla bllip-sm 922 bnc-brown 7 1702"

|======================================================================================            | 88% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 7 24515"

|=======================================================================================           | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 7 3102"
[1] "vanilla bllip-xs 111 bnc-brown 7 1698"

|=========================================================================================         | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 dundee 7 24531"

|==========================================================================================        | 93% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 7 3100"
[1] "vanilla bllip-xs 120 bnc-brown 7 1664"

|============================================================================================      | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 7 24499"

|=============================================================================================     | 96% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 7 3112"
[1] "vanilla bllip-xs 922 bnc-brown 7 1672"

|===============================================================================================   | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 7 24639"

|================================================================================================  | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 7 3106"
[1] "5gram bllip-lg 1111 bnc-brown 8 1706"
[1] "5gram bllip-lg 1111 dundee 8 24557"
[1] "5gram bllip-lg 1111 natural-stories 8 3085"
[1] "5gram bllip-md 1111 bnc-brown 8 1698"
[1] "5gram bllip-md 1111 dundee 8 24609"
[1] "5gram bllip-md 1111 natural-stories 8 3140"
[1] "5gram bllip-sm 1111 bnc-brown 8 1675"
[1] "5gram bllip-sm 1111 dundee 8 24523"
[1] "5gram bllip-sm 1111 natural-stories 8 3122"
[1] "5gram bllip-xs 1111 bnc-brown 8 1695"
[1] "5gram bllip-xs 1111 dundee 8 24484"
[1] "5gram bllip-xs 1111 natural-stories 8 3082"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 8 1668"
[1] "gpt2 bllip-lg 1587139950 dundee 8 24619"
[1] "gpt2 bllip-lg 1587139950 natural-stories 8 3064"
[1] "gpt2 bllip-lg-gptbpe 1581955288 bnc-brown 8 1696"
[1] "gpt2 bllip-lg-gptbpe 1581955288 dundee 8 24564"
[1] "gpt2 bllip-lg-gptbpe 1581955288 natural-stories 8 3098"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 8 1662"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 8 24519"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 8 3117"
[1] "gpt2 bllip-md 1586986276 bnc-brown 8 1691"
[1] "gpt2 bllip-md 1586986276 dundee 8 24535"
[1] "gpt2 bllip-md 1586986276 natural-stories 8 3093"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 8 24599"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 8 3103"
[1] "gpt2 bllip-md-gptbpe 1581861474 bnc-brown 8 1707"
[1] "gpt2 bllip-md-gptbpe 1581861474 dundee 8 24521"
[1] "gpt2 bllip-md-gptbpe 1581861474 natural-stories 8 3107"
[1] "gpt2 bllip-md-gptbpe 1582126320 bnc-brown 8 1685"
[1] "gpt2 bllip-md-gptbpe 1582126320 dundee 8 24510"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 8 1659"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 8 24531"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 8 3118"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 8 1714"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 8 24549"

|=====================================                                                             | 38% ~3 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 8 3083"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 8 1701"

|=======================================                                                           | 40% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 8 24542"

|========================================                                                          | 41% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 8 3079"
[1] "rnng bllip-lg 7245 bnc-brown 8 1673"

|==========================================                                                        | 43% ~3 s remaining     [1] "rnng bllip-lg 7245 dundee 8 24514"

|===========================================                                                       | 44% ~3 s remaining     [1] "rnng bllip-lg 7245 natural-stories 8 3087"

|============================================                                                      | 45% ~3 s remaining     [1] "rnng bllip-md 3602 bnc-brown 8 1707"

|=============================================                                                     | 46% ~3 s remaining     [1] "rnng bllip-md 3602 dundee 8 24537"

|==============================================                                                    | 47% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 8 3128"
[1] "rnng bllip-md 44862 bnc-brown 8 1684"

|================================================                                                  | 49% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 8 24526"

|=================================================                                                 | 51% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 8 3112"
[1] "rnng bllip-sm 7877 bnc-brown 8 1678"

|===================================================                                               | 53% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 8 24594"

|====================================================                                              | 54% ~2 s remaining     [1] "rnng bllip-sm 7877 natural-stories 8 3115"
[1] "rnng bllip-sm 64924 bnc-brown 8 1682"

|======================================================                                            | 56% ~2 s remaining     [1] "rnng bllip-sm 64924 dundee 8 24480"

|=======================================================                                           | 57% ~2 s remaining     [1] "rnng bllip-sm 64924 natural-stories 8 3058"
[1] "rnng bllip-xs 4301 bnc-brown 8 1693"

|=========================================================                                         | 59% ~2 s remaining     [1] "rnng bllip-xs 4301 dundee 8 24599"

|==========================================================                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 8 3118"
[1] "rnng bllip-xs 28066 bnc-brown 8 1703"

|============================================================                                      | 62% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 8 24573"

|=============================================================                                     | 63% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 8 3117"
[1] "rnng bllip-xs 28068 bnc-brown 8 1683"

|===============================================================                                   | 65% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 8 24611"

|================================================================                                  | 66% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 8 3074"
[1] "rnng bllip-xs 51272 bnc-brown 8 1706"

|===================================================================                               | 68% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 8 24534"

|====================================================================                              | 69% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 8 3112"
[1] "vanilla bllip-lg 111 bnc-brown 8 1693"

|======================================================================                            | 72% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 8 24559"

|=======================================================================                           | 73% ~1 s remaining     [1] "vanilla bllip-lg 111 natural-stories 8 3103"
[1] "vanilla bllip-md 120 bnc-brown 8 1705"

|=========================================================================                         | 75% ~1 s remaining     [1] "vanilla bllip-md 120 dundee 8 24553"

|==========================================================================                        | 76% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 8 3100"
[1] "vanilla bllip-md 607 bnc-brown 8 1677"

|============================================================================                      | 78% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 8 1668"

|=============================================================================                     | 79% ~1 s remaining     [1] "vanilla bllip-md 922 dundee 8 24570"

|==============================================================================                    | 80% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 8 3098"
[1] "vanilla bllip-sm 111 bnc-brown 8 1690"

|================================================================================                  | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 8 24538"

|=================================================================================                 | 83% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 8 3088"
[1] "vanilla bllip-sm 120 bnc-brown 8 1704"

|===================================================================================               | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 8 24535"

|====================================================================================              | 86% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 8 3101"
[1] "vanilla bllip-sm 922 bnc-brown 8 1696"

|======================================================================================            | 88% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 8 24520"

|=======================================================================================           | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 8 3068"
[1] "vanilla bllip-xs 111 bnc-brown 8 1710"

|=========================================================================================         | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 dundee 8 24521"

|==========================================================================================        | 93% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 8 3081"
[1] "vanilla bllip-xs 120 bnc-brown 8 1716"

|============================================================================================      | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 8 24552"

|=============================================================================================     | 96% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 8 3094"

|==============================================================================================    | 97% ~0 s remaining     [1] "vanilla bllip-xs 922 bnc-brown 8 1710"
[1] "vanilla bllip-xs 922 dundee 8 24510"

|================================================================================================  | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 8 3100"
[1] "5gram bllip-lg 1111 bnc-brown 8 1706"
[1] "5gram bllip-lg 1111 dundee 8 24557"
[1] "5gram bllip-lg 1111 natural-stories 8 3085"
[1] "5gram bllip-md 1111 bnc-brown 8 1698"
[1] "5gram bllip-md 1111 dundee 8 24609"
[1] "5gram bllip-md 1111 natural-stories 8 3140"
[1] "5gram bllip-sm 1111 bnc-brown 8 1675"
[1] "5gram bllip-sm 1111 dundee 8 24523"
[1] "5gram bllip-sm 1111 natural-stories 8 3122"
[1] "5gram bllip-xs 1111 bnc-brown 8 1695"
[1] "5gram bllip-xs 1111 dundee 8 24484"
[1] "5gram bllip-xs 1111 natural-stories 8 3082"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 8 1668"
[1] "gpt2 bllip-lg 1587139950 dundee 8 24619"
[1] "gpt2 bllip-lg 1587139950 natural-stories 8 3064"
[1] "gpt2 bllip-lg-gptbpe 1581955288 bnc-brown 8 1696"
[1] "gpt2 bllip-lg-gptbpe 1581955288 dundee 8 24564"
[1] "gpt2 bllip-lg-gptbpe 1581955288 natural-stories 8 3098"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 8 1662"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 8 24519"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 8 3117"
[1] "gpt2 bllip-md 1586986276 bnc-brown 8 1691"
[1] "gpt2 bllip-md 1586986276 dundee 8 24535"
[1] "gpt2 bllip-md 1586986276 natural-stories 8 3093"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 8 24599"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 8 3103"
[1] "gpt2 bllip-md-gptbpe 1581861474 bnc-brown 8 1707"
[1] "gpt2 bllip-md-gptbpe 1581861474 dundee 8 24521"
[1] "gpt2 bllip-md-gptbpe 1581861474 natural-stories 8 3107"
[1] "gpt2 bllip-md-gptbpe 1582126320 bnc-brown 8 1685"
[1] "gpt2 bllip-md-gptbpe 1582126320 dundee 8 24510"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 8 1659"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 8 24531"

|==================================                                                                | 35% ~4 s remaining     [1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 8 3118"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 8 1714"

|====================================                                                              | 37% ~4 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 8 24549"

|=====================================                                                             | 38% ~4 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 8 3083"

|======================================                                                            | 39% ~4 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 8 1701"
[1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 8 24542"

|========================================                                                          | 41% ~4 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 8 3079"
[1] "rnng bllip-lg 7245 bnc-brown 8 1673"

|==========================================                                                        | 43% ~3 s remaining     [1] "rnng bllip-lg 7245 dundee 8 24514"

|===========================================                                                       | 44% ~3 s remaining     [1] "rnng bllip-lg 7245 natural-stories 8 3087"
[1] "rnng bllip-md 3602 bnc-brown 8 1707"

|=============================================                                                     | 46% ~3 s remaining     [1] "rnng bllip-md 3602 dundee 8 24537"

|==============================================                                                    | 47% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 8 3128"
[1] "rnng bllip-md 44862 bnc-brown 8 1684"

|================================================                                                  | 49% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 8 24526"

|=================================================                                                 | 51% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 8 3112"
[1] "rnng bllip-sm 7877 bnc-brown 8 1678"

|===================================================                                               | 53% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 8 24594"

|====================================================                                              | 54% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 8 3115"
[1] "rnng bllip-sm 64924 bnc-brown 8 1682"

|======================================================                                            | 56% ~3 s remaining     [1] "rnng bllip-sm 64924 dundee 8 24480"

|=======================================================                                           | 57% ~3 s remaining     [1] "rnng bllip-sm 64924 natural-stories 8 3058"
[1] "rnng bllip-xs 4301 bnc-brown 8 1693"

|=========================================================                                         | 59% ~2 s remaining     [1] "rnng bllip-xs 4301 dundee 8 24599"

|==========================================================                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 8 3118"
[1] "rnng bllip-xs 28066 bnc-brown 8 1703"

|============================================================                                      | 62% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 8 24573"

|=============================================================                                     | 63% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 8 3117"
[1] "rnng bllip-xs 28068 bnc-brown 8 1683"

|===============================================================                                   | 65% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 8 24611"

|================================================================                                  | 66% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 8 3074"
[1] "rnng bllip-xs 51272 bnc-brown 8 1706"

|===================================================================                               | 68% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 8 24534"

|====================================================================                              | 69% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 8 3112"
[1] "vanilla bllip-lg 111 bnc-brown 8 1693"

|======================================================================                            | 72% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 8 24559"

|=======================================================================                           | 73% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 8 3103"
[1] "vanilla bllip-md 120 bnc-brown 8 1705"

|=========================================================================                         | 75% ~2 s remaining     [1] "vanilla bllip-md 120 dundee 8 24553"

|==========================================================================                        | 76% ~2 s remaining     [1] "vanilla bllip-md 120 natural-stories 8 3100"
[1] "vanilla bllip-md 607 bnc-brown 8 1677"

|============================================================================                      | 78% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 8 1668"
[1] "vanilla bllip-md 922 dundee 8 24570"

|==============================================================================                    | 80% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 8 3098"
[1] "vanilla bllip-sm 111 bnc-brown 8 1690"

|================================================================================                  | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 8 24538"

|=================================================================================                 | 83% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 8 3088"
[1] "vanilla bllip-sm 120 bnc-brown 8 1704"

|===================================================================================               | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 8 24535"

|====================================================================================              | 86% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 8 3101"
[1] "vanilla bllip-sm 922 bnc-brown 8 1696"

|======================================================================================            | 88% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 8 24520"

|=======================================================================================           | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 8 3068"
[1] "vanilla bllip-xs 111 bnc-brown 8 1710"

|=========================================================================================         | 92% ~1 s remaining     [1] "vanilla bllip-xs 111 dundee 8 24521"

|==========================================================================================        | 93% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 8 3081"
[1] "vanilla bllip-xs 120 bnc-brown 8 1716"

|============================================================================================      | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 8 24552"

|=============================================================================================     | 96% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 8 3094"
[1] "vanilla bllip-xs 922 bnc-brown 8 1710"

|===============================================================================================   | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 8 24510"

|================================================================================================  | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 8 3100"
[1] "5gram bllip-lg 1111 bnc-brown 9 1676"
[1] "5gram bllip-lg 1111 dundee 9 24555"
[1] "5gram bllip-lg 1111 natural-stories 9 3061"
[1] "5gram bllip-md 1111 bnc-brown 9 1696"
[1] "5gram bllip-md 1111 dundee 9 24522"
[1] "5gram bllip-md 1111 natural-stories 9 3098"
[1] "5gram bllip-sm 1111 bnc-brown 9 1680"
[1] "5gram bllip-sm 1111 dundee 9 24468"
[1] "5gram bllip-sm 1111 natural-stories 9 3083"
[1] "5gram bllip-xs 1111 bnc-brown 9 1694"
[1] "5gram bllip-xs 1111 dundee 9 24489"
[1] "5gram bllip-xs 1111 natural-stories 9 3085"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 9 1689"
[1] "gpt2 bllip-lg 1587139950 dundee 9 24471"
[1] "gpt2 bllip-lg 1587139950 natural-stories 9 3121"
[1] "gpt2 bllip-lg-gptbpe 1581955288 bnc-brown 9 1701"
[1] "gpt2 bllip-lg-gptbpe 1581955288 dundee 9 24527"
[1] "gpt2 bllip-lg-gptbpe 1581955288 natural-stories 9 3112"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 9 1728"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 9 24523"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 9 3096"
[1] "gpt2 bllip-md 1586986276 bnc-brown 9 1695"
[1] "gpt2 bllip-md 1586986276 dundee 9 24595"
[1] "gpt2 bllip-md 1586986276 natural-stories 9 3108"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 9 24560"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 9 3085"
[1] "gpt2 bllip-md-gptbpe 1581861474 bnc-brown 9 1697"
[1] "gpt2 bllip-md-gptbpe 1581861474 dundee 9 24579"
[1] "gpt2 bllip-md-gptbpe 1581861474 natural-stories 9 3099"
[1] "gpt2 bllip-md-gptbpe 1582126320 bnc-brown 9 1669"
[1] "gpt2 bllip-md-gptbpe 1582126320 dundee 9 24575"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 9 1670"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 9 24607"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 9 3103"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 9 1704"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 9 24547"

|=====================================                                                             | 38% ~3 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 9 3134"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 9 1685"

|=======================================                                                           | 40% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 9 24539"

|========================================                                                          | 41% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 9 3116"
[1] "rnng bllip-lg 7245 bnc-brown 9 1702"

|==========================================                                                        | 43% ~3 s remaining     [1] "rnng bllip-lg 7245 dundee 9 24560"

|===========================================                                                       | 44% ~3 s remaining     [1] "rnng bllip-lg 7245 natural-stories 9 3082"
[1] "rnng bllip-md 3602 bnc-brown 9 1678"

|=============================================                                                     | 46% ~3 s remaining     [1] "rnng bllip-md 3602 dundee 9 24630"

|==============================================                                                    | 47% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 9 3105"
[1] "rnng bllip-md 44862 bnc-brown 9 1688"

|================================================                                                  | 49% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 9 24632"

|=================================================                                                 | 51% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 9 3131"
[1] "rnng bllip-sm 7877 bnc-brown 9 1679"

|===================================================                                               | 53% ~2 s remaining     [1] "rnng bllip-sm 7877 dundee 9 24465"

|====================================================                                              | 54% ~2 s remaining     [1] "rnng bllip-sm 7877 natural-stories 9 3112"

|=====================================================                                             | 55% ~2 s remaining     [1] "rnng bllip-sm 64924 bnc-brown 9 1690"
[1] "rnng bllip-sm 64924 dundee 9 24596"

|=======================================================                                           | 57% ~2 s remaining     [1] "rnng bllip-sm 64924 natural-stories 9 3099"
[1] "rnng bllip-xs 4301 bnc-brown 9 1706"

|=========================================================                                         | 59% ~2 s remaining     [1] "rnng bllip-xs 4301 dundee 9 24449"

|==========================================================                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 9 3078"
[1] "rnng bllip-xs 28066 bnc-brown 9 1689"

|============================================================                                      | 62% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 9 24549"

|=============================================================                                     | 63% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 9 3078"
[1] "rnng bllip-xs 28068 bnc-brown 9 1691"

|===============================================================                                   | 65% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 9 24477"

|================================================================                                  | 66% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 9 3101"
[1] "rnng bllip-xs 51272 bnc-brown 9 1696"

|===================================================================                               | 68% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 9 24467"

|====================================================================                              | 69% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 9 3134"
[1] "vanilla bllip-lg 111 bnc-brown 9 1701"

|======================================================================                            | 72% ~1 s remaining     [1] "vanilla bllip-lg 111 dundee 9 24610"

|=======================================================================                           | 73% ~1 s remaining     [1] "vanilla bllip-lg 111 natural-stories 9 3096"
[1] "vanilla bllip-md 120 bnc-brown 9 1699"

|=========================================================================                         | 75% ~1 s remaining     [1] "vanilla bllip-md 120 dundee 9 24485"

|==========================================================================                        | 76% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 9 3087"
[1] "vanilla bllip-md 607 bnc-brown 9 1708"

|============================================================================                      | 78% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 9 1697"
[1] "vanilla bllip-md 922 dundee 9 24558"

|==============================================================================                    | 80% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 9 3092"
[1] "vanilla bllip-sm 111 bnc-brown 9 1670"

|================================================================================                  | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 9 24542"

|=================================================================================                 | 83% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 9 3078"
[1] "vanilla bllip-sm 120 bnc-brown 9 1705"

|===================================================================================               | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 9 24547"

|====================================================================================              | 86% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 9 3124"
[1] "vanilla bllip-sm 922 bnc-brown 9 1723"

|======================================================================================            | 88% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 9 24622"

|=======================================================================================           | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 9 3089"
[1] "vanilla bllip-xs 111 bnc-brown 9 1698"

|=========================================================================================         | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 dundee 9 24458"

|==========================================================================================        | 93% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 9 3098"
[1] "vanilla bllip-xs 120 bnc-brown 9 1675"

|============================================================================================      | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 9 24690"

|=============================================================================================     | 96% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 9 3067"
[1] "vanilla bllip-xs 922 bnc-brown 9 1707"

|===============================================================================================   | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 9 24551"

|================================================================================================  | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 9 3085"
[1] "5gram bllip-lg 1111 bnc-brown 9 1676"
[1] "5gram bllip-lg 1111 dundee 9 24555"
[1] "5gram bllip-lg 1111 natural-stories 9 3061"
[1] "5gram bllip-md 1111 bnc-brown 9 1696"
[1] "5gram bllip-md 1111 dundee 9 24522"
[1] "5gram bllip-md 1111 natural-stories 9 3098"
[1] "5gram bllip-sm 1111 bnc-brown 9 1680"
[1] "5gram bllip-sm 1111 dundee 9 24468"
[1] "5gram bllip-sm 1111 natural-stories 9 3083"
[1] "5gram bllip-xs 1111 bnc-brown 9 1694"
[1] "5gram bllip-xs 1111 dundee 9 24489"
[1] "5gram bllip-xs 1111 natural-stories 9 3085"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 9 1689"
[1] "gpt2 bllip-lg 1587139950 dundee 9 24471"
[1] "gpt2 bllip-lg 1587139950 natural-stories 9 3121"
[1] "gpt2 bllip-lg-gptbpe 1581955288 bnc-brown 9 1701"
[1] "gpt2 bllip-lg-gptbpe 1581955288 dundee 9 24527"
[1] "gpt2 bllip-lg-gptbpe 1581955288 natural-stories 9 3112"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 9 1728"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 9 24523"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 9 3096"
[1] "gpt2 bllip-md 1586986276 bnc-brown 9 1695"
[1] "gpt2 bllip-md 1586986276 dundee 9 24595"
[1] "gpt2 bllip-md 1586986276 natural-stories 9 3108"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 9 24560"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 9 3085"
[1] "gpt2 bllip-md-gptbpe 1581861474 bnc-brown 9 1697"
[1] "gpt2 bllip-md-gptbpe 1581861474 dundee 9 24579"
[1] "gpt2 bllip-md-gptbpe 1581861474 natural-stories 9 3099"
[1] "gpt2 bllip-md-gptbpe 1582126320 bnc-brown 9 1669"
[1] "gpt2 bllip-md-gptbpe 1582126320 dundee 9 24575"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 9 1670"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 9 24607"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 9 3103"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 9 1704"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 9 24547"

|=====================================                                                             | 38% ~3 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 9 3134"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 9 1685"

|=======================================                                                           | 40% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 9 24539"

|========================================                                                          | 41% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 9 3116"
[1] "rnng bllip-lg 7245 bnc-brown 9 1702"

|==========================================                                                        | 43% ~3 s remaining     [1] "rnng bllip-lg 7245 dundee 9 24560"

|===========================================                                                       | 44% ~3 s remaining     [1] "rnng bllip-lg 7245 natural-stories 9 3082"
[1] "rnng bllip-md 3602 bnc-brown 9 1678"

|=============================================                                                     | 46% ~3 s remaining     [1] "rnng bllip-md 3602 dundee 9 24630"

|==============================================                                                    | 47% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 9 3105"
[1] "rnng bllip-md 44862 bnc-brown 9 1688"

|================================================                                                  | 49% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 9 24632"

|=================================================                                                 | 51% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 9 3131"
[1] "rnng bllip-sm 7877 bnc-brown 9 1679"

|===================================================                                               | 53% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 9 24465"

|====================================================                                              | 54% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 9 3112"
[1] "rnng bllip-sm 64924 bnc-brown 9 1690"

|======================================================                                            | 56% ~2 s remaining     [1] "rnng bllip-sm 64924 dundee 9 24596"

|=======================================================                                           | 57% ~2 s remaining     [1] "rnng bllip-sm 64924 natural-stories 9 3099"
[1] "rnng bllip-xs 4301 bnc-brown 9 1706"

|=========================================================                                         | 59% ~2 s remaining     [1] "rnng bllip-xs 4301 dundee 9 24449"

|==========================================================                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 9 3078"
[1] "rnng bllip-xs 28066 bnc-brown 9 1689"

|============================================================                                      | 62% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 9 24549"

|=============================================================                                     | 63% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 9 3078"
[1] "rnng bllip-xs 28068 bnc-brown 9 1691"

|===============================================================                                   | 65% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 9 24477"

|================================================================                                  | 66% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 9 3101"
[1] "rnng bllip-xs 51272 bnc-brown 9 1696"

|===================================================================                               | 68% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 9 24467"

|====================================================================                              | 69% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 9 3134"
[1] "vanilla bllip-lg 111 bnc-brown 9 1701"

|======================================================================                            | 72% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 9 24610"

|=======================================================================                           | 73% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 9 3096"
[1] "vanilla bllip-md 120 bnc-brown 9 1699"

|=========================================================================                         | 75% ~1 s remaining     [1] "vanilla bllip-md 120 dundee 9 24485"

|==========================================================================                        | 76% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 9 3087"
[1] "vanilla bllip-md 607 bnc-brown 9 1708"

|============================================================================                      | 78% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 9 1697"
[1] "vanilla bllip-md 922 dundee 9 24558"

|==============================================================================                    | 80% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 9 3092"
[1] "vanilla bllip-sm 111 bnc-brown 9 1670"

|================================================================================                  | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 9 24542"

|=================================================================================                 | 83% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 9 3078"

|==================================================================================                | 84% ~1 s remaining     [1] "vanilla bllip-sm 120 bnc-brown 9 1705"
[1] "vanilla bllip-sm 120 dundee 9 24547"

|====================================================================================              | 86% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 9 3124"
[1] "vanilla bllip-sm 922 bnc-brown 9 1723"

|======================================================================================            | 88% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 9 24622"

|=======================================================================================           | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 9 3089"
[1] "vanilla bllip-xs 111 bnc-brown 9 1698"

|=========================================================================================         | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 dundee 9 24458"

|==========================================================================================        | 93% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 9 3098"

|===========================================================================================       | 94% ~0 s remaining     [1] "vanilla bllip-xs 120 bnc-brown 9 1675"
[1] "vanilla bllip-xs 120 dundee 9 24690"

|=============================================================================================     | 96% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 9 3067"
[1] "vanilla bllip-xs 922 bnc-brown 9 1707"

|===============================================================================================   | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 9 24551"

|================================================================================================  | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 9 3085"
[1] "5gram bllip-lg 1111 bnc-brown 10 1689"
[1] "5gram bllip-lg 1111 dundee 10 24499"
[1] "5gram bllip-lg 1111 natural-stories 10 3099"
[1] "5gram bllip-md 1111 bnc-brown 10 1705"
[1] "5gram bllip-md 1111 dundee 10 24568"
[1] "5gram bllip-md 1111 natural-stories 10 3089"
[1] "5gram bllip-sm 1111 bnc-brown 10 1703"
[1] "5gram bllip-sm 1111 dundee 10 24479"
[1] "5gram bllip-sm 1111 natural-stories 10 3048"
[1] "5gram bllip-xs 1111 bnc-brown 10 1696"
[1] "5gram bllip-xs 1111 dundee 10 24586"
[1] "5gram bllip-xs 1111 natural-stories 10 3112"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 10 1686"
[1] "gpt2 bllip-lg 1587139950 dundee 10 24537"
[1] "gpt2 bllip-lg 1587139950 natural-stories 10 3091"
[1] "gpt2 bllip-lg-gptbpe 1581955288 bnc-brown 10 1696"
[1] "gpt2 bllip-lg-gptbpe 1581955288 dundee 10 24586"
[1] "gpt2 bllip-lg-gptbpe 1581955288 natural-stories 10 3119"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 10 1670"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 10 24616"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 10 3089"
[1] "gpt2 bllip-md 1586986276 bnc-brown 10 1706"
[1] "gpt2 bllip-md 1586986276 dundee 10 24504"
[1] "gpt2 bllip-md 1586986276 natural-stories 10 3089"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 10 24396"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 10 3127"
[1] "gpt2 bllip-md-gptbpe 1581861474 bnc-brown 10 1690"
[1] "gpt2 bllip-md-gptbpe 1581861474 dundee 10 24495"
[1] "gpt2 bllip-md-gptbpe 1581861474 natural-stories 10 3116"
[1] "gpt2 bllip-md-gptbpe 1582126320 bnc-brown 10 1713"
[1] "gpt2 bllip-md-gptbpe 1582126320 dundee 10 24646"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 10 1714"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 10 24599"

|==================================                                                                | 35% ~4 s remaining     [1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 10 3100"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 10 1701"

|====================================                                                              | 37% ~4 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 10 24483"

|=====================================                                                             | 38% ~4 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 10 3094"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 10 1708"

|=======================================                                                           | 40% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 10 24480"

|========================================                                                          | 41% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 10 3073"
[1] "rnng bllip-lg 7245 bnc-brown 10 1701"

|==========================================                                                        | 43% ~3 s remaining     [1] "rnng bllip-lg 7245 dundee 10 24561"

|===========================================                                                       | 44% ~4 s remaining     [1] "rnng bllip-lg 7245 natural-stories 10 3095"
[1] "rnng bllip-md 3602 bnc-brown 10 1711"

|=============================================                                                     | 46% ~4 s remaining     [1] "rnng bllip-md 3602 dundee 10 24534"

|==============================================                                                    | 47% ~4 s remaining     [1] "rnng bllip-md 3602 natural-stories 10 3080"
[1] "rnng bllip-md 44862 bnc-brown 10 1679"

|================================================                                                  | 49% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 10 24548"

|=================================================                                                 | 51% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 10 3088"
[1] "rnng bllip-sm 7877 bnc-brown 10 1700"

|===================================================                                               | 53% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 10 24580"

|====================================================                                              | 54% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 10 3094"
[1] "rnng bllip-sm 64924 bnc-brown 10 1694"

|======================================================                                            | 56% ~3 s remaining     [1] "rnng bllip-sm 64924 dundee 10 24474"

|=======================================================                                           | 57% ~3 s remaining     [1] "rnng bllip-sm 64924 natural-stories 10 3109"
[1] "rnng bllip-xs 4301 bnc-brown 10 1696"

|=========================================================                                         | 59% ~3 s remaining     [1] "rnng bllip-xs 4301 dundee 10 24563"

|==========================================================                                        | 60% ~3 s remaining     [1] "rnng bllip-xs 4301 natural-stories 10 3129"
[1] "rnng bllip-xs 28066 bnc-brown 10 1686"

|============================================================                                      | 62% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 10 24555"

|=============================================================                                     | 63% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 10 3114"
[1] "rnng bllip-xs 28068 bnc-brown 10 1703"

|===============================================================                                   | 65% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 10 24522"

|================================================================                                  | 66% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 10 3084"
[1] "rnng bllip-xs 51272 bnc-brown 10 1681"

|===================================================================                               | 68% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 10 24564"

|====================================================================                              | 69% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 10 3085"
[1] "vanilla bllip-lg 111 bnc-brown 10 1690"

|======================================================================                            | 72% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 10 24560"

|=======================================================================                           | 73% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 10 3083"
[1] "vanilla bllip-md 120 bnc-brown 10 1688"

|=========================================================================                         | 75% ~2 s remaining     [1] "vanilla bllip-md 120 dundee 10 24636"

|==========================================================================                        | 76% ~2 s remaining     [1] "vanilla bllip-md 120 natural-stories 10 3107"
[1] "vanilla bllip-md 607 bnc-brown 10 1712"

|============================================================================                      | 78% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 10 1681"
[1] "vanilla bllip-md 922 dundee 10 24596"

|==============================================================================                    | 80% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 10 3128"
[1] "vanilla bllip-sm 111 bnc-brown 10 1699"

|================================================================================                  | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 10 24567"

|=================================================================================                 | 83% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 10 3105"
[1] "vanilla bllip-sm 120 bnc-brown 10 1686"

|===================================================================================               | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 10 24573"

|====================================================================================              | 86% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 10 3073"
[1] "vanilla bllip-sm 922 bnc-brown 10 1690"

|======================================================================================            | 88% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 10 24521"

|=======================================================================================           | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 10 3087"
[1] "vanilla bllip-xs 111 bnc-brown 10 1706"

|=========================================================================================         | 92% ~1 s remaining     [1] "vanilla bllip-xs 111 dundee 10 24609"

|==========================================================================================        | 93% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 10 3070"
[1] "vanilla bllip-xs 120 bnc-brown 10 1714"

|============================================================================================      | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 10 24453"

|=============================================================================================     | 96% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 10 3105"
[1] "vanilla bllip-xs 922 bnc-brown 10 1711"

|===============================================================================================   | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 10 24516"

|================================================================================================  | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 10 3075"
[1] "5gram bllip-lg 1111 bnc-brown 10 1689"
[1] "5gram bllip-lg 1111 dundee 10 24499"
[1] "5gram bllip-lg 1111 natural-stories 10 3099"
[1] "5gram bllip-md 1111 bnc-brown 10 1705"
[1] "5gram bllip-md 1111 dundee 10 24568"
[1] "5gram bllip-md 1111 natural-stories 10 3089"
[1] "5gram bllip-sm 1111 bnc-brown 10 1703"
[1] "5gram bllip-sm 1111 dundee 10 24479"
[1] "5gram bllip-sm 1111 natural-stories 10 3048"
[1] "5gram bllip-xs 1111 bnc-brown 10 1696"
[1] "5gram bllip-xs 1111 dundee 10 24586"
[1] "5gram bllip-xs 1111 natural-stories 10 3112"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 10 1686"
[1] "gpt2 bllip-lg 1587139950 dundee 10 24537"
[1] "gpt2 bllip-lg 1587139950 natural-stories 10 3091"
[1] "gpt2 bllip-lg-gptbpe 1581955288 bnc-brown 10 1696"
[1] "gpt2 bllip-lg-gptbpe 1581955288 dundee 10 24586"
[1] "gpt2 bllip-lg-gptbpe 1581955288 natural-stories 10 3119"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 10 1670"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 10 24616"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 10 3089"
[1] "gpt2 bllip-md 1586986276 bnc-brown 10 1706"
[1] "gpt2 bllip-md 1586986276 dundee 10 24504"
[1] "gpt2 bllip-md 1586986276 natural-stories 10 3089"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 10 24396"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 10 3127"
[1] "gpt2 bllip-md-gptbpe 1581861474 bnc-brown 10 1690"
[1] "gpt2 bllip-md-gptbpe 1581861474 dundee 10 24495"
[1] "gpt2 bllip-md-gptbpe 1581861474 natural-stories 10 3116"
[1] "gpt2 bllip-md-gptbpe 1582126320 bnc-brown 10 1713"
[1] "gpt2 bllip-md-gptbpe 1582126320 dundee 10 24646"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 10 1714"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 10 24599"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 10 3100"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 10 1701"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 10 24483"

|=====================================                                                             | 38% ~3 s remaining     [1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 10 3094"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 10 1708"

|=======================================                                                           | 40% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 10 24480"

|========================================                                                          | 41% ~3 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 10 3073"
[1] "rnng bllip-lg 7245 bnc-brown 10 1701"

|==========================================                                                        | 43% ~3 s remaining     [1] "rnng bllip-lg 7245 dundee 10 24561"

|===========================================                                                       | 44% ~3 s remaining     [1] "rnng bllip-lg 7245 natural-stories 10 3095"
[1] "rnng bllip-md 3602 bnc-brown 10 1711"

|=============================================                                                     | 46% ~3 s remaining     [1] "rnng bllip-md 3602 dundee 10 24534"

|==============================================                                                    | 47% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 10 3080"
[1] "rnng bllip-md 44862 bnc-brown 10 1679"

|================================================                                                  | 49% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 10 24548"

|=================================================                                                 | 51% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 10 3088"
[1] "rnng bllip-sm 7877 bnc-brown 10 1700"

|===================================================                                               | 53% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 10 24580"

|====================================================                                              | 54% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 10 3094"
[1] "rnng bllip-sm 64924 bnc-brown 10 1694"

|======================================================                                            | 56% ~2 s remaining     [1] "rnng bllip-sm 64924 dundee 10 24474"

|=======================================================                                           | 57% ~2 s remaining     [1] "rnng bllip-sm 64924 natural-stories 10 3109"
[1] "rnng bllip-xs 4301 bnc-brown 10 1696"

|=========================================================                                         | 59% ~2 s remaining     [1] "rnng bllip-xs 4301 dundee 10 24563"

|==========================================================                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 10 3129"
[1] "rnng bllip-xs 28066 bnc-brown 10 1686"

|============================================================                                      | 62% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 10 24555"

|=============================================================                                     | 63% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 10 3114"
[1] "rnng bllip-xs 28068 bnc-brown 10 1703"

|===============================================================                                   | 65% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 10 24522"

|================================================================                                  | 66% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 10 3084"
[1] "rnng bllip-xs 51272 bnc-brown 10 1681"

|===================================================================                               | 68% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 10 24564"

|====================================================================                              | 69% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 10 3085"
[1] "vanilla bllip-lg 111 bnc-brown 10 1690"

|======================================================================                            | 72% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 10 24560"

|=======================================================================                           | 73% ~1 s remaining     [1] "vanilla bllip-lg 111 natural-stories 10 3083"
[1] "vanilla bllip-md 120 bnc-brown 10 1688"

|=========================================================================                         | 75% ~1 s remaining     [1] "vanilla bllip-md 120 dundee 10 24636"

|==========================================================================                        | 76% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 10 3107"
[1] "vanilla bllip-md 607 bnc-brown 10 1712"

|============================================================================                      | 78% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 10 1681"
[1] "vanilla bllip-md 922 dundee 10 24596"

|==============================================================================                    | 80% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 10 3128"
[1] "vanilla bllip-sm 111 bnc-brown 10 1699"

|================================================================================                  | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 10 24567"

|=================================================================================                 | 83% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 10 3105"
[1] "vanilla bllip-sm 120 bnc-brown 10 1686"

|===================================================================================               | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 10 24573"

|====================================================================================              | 86% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 10 3073"
[1] "vanilla bllip-sm 922 bnc-brown 10 1690"

|======================================================================================            | 88% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 10 24521"

|=======================================================================================           | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 10 3087"
[1] "vanilla bllip-xs 111 bnc-brown 10 1706"

|=========================================================================================         | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 dundee 10 24609"

|==========================================================================================        | 93% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 10 3070"
[1] "vanilla bllip-xs 120 bnc-brown 10 1714"

|============================================================================================      | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 10 24453"

|=============================================================================================     | 96% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 10 3105"
[1] "vanilla bllip-xs 922 bnc-brown 10 1711"

|===============================================================================================   | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 10 24516"

|================================================================================================  | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 10 3075"
#write.csv(full_residuals, "../data/analysis_checkpoints/full_residuals.csv")
#write.csv(baseline_residuals, "../data/analysis_checkpoints/baseline_residuals.csv")
model_deltas = log_lik_deltas %>%
  group_by(model, training, seed, corpus) %>% 
  summarise(mean_delta_log_lik = mean(delta_log_lik),
            sem_delta_log_lik = sd(delta_log_lik) / sqrt(length(delta_log_lik)))
`summarise()` has grouped output by 'model', 'training', 'seed'. You can override using the `.groups` argument.
write.csv(full_model_results, "../data/analysis_checkpoints/full_model_result.csv")
write.csv(baseline_results, "../data/analysis_checkpoints/baseline_results.csv")
#full_model_results = read.csv("../data/analysis_checkpoints/ffull_model_results.csv")
#baseline_results = read.csv("../data/analysis_checkpoints/fbaseline_resultsb.csv")
metric <- "ΔLogLik"
#metric <- "-ΔMSE"

# # Select the relevant metric.
model_deltas = model_deltas %>%
    # Retrieve the current test metric
    mutate(delta_test_mean = mean_delta_log_lik,
           delta_test_sem = sem_delta_log_lik) %>%
    # mutate(delta_test_mean = mean_delta_mse,
    #        delta_test_sem = sem_delta_mse)
    
    # Remove the raw metrics.
    select(-mean_delta_log_lik, -sem_delta_log_lik,
           #-mean_delta_mse, -sem_delta_mse
           )
model_deltas
# Sanity check: training on train+test data should yield improved performance over training on just training data. (When evaluating on test data.)
 full_baselines = all_data %>%
   group_by(model, training, seed, corpus) %>%
   summarise(baseline_train_all_test_lik = logLik_test(lm(psychometric ~ len + freq + sent_pos, data=.), semi_join(test_data, ., by=c("training", "model", "seed", "corpus")), semi_join(test_data, ., by=c("training", "model", "seed", "corpus"))$psychometric)) %>%
   ungroup()
 full_baselines
 
 full_baselines %>%
   right_join(baselines, by=c("seed", "training", "model", "corpus")) %>%
   mutate(delta=baseline_train_all_test_lik-baseline_test_lik) %>%
   select(-baseline_lik) # %>%
   #select(-baseline_test_lik, -baseline_train_all_test_lik, -baseline_lik, -baseline_test_mse)

Load language model data (SyntaxGym, PPL)

language_model_data = read.csv("../data/model_metadata.csv") %>%
  mutate(model = as.character(model),
         model = if_else(model == "gpt-2", "gpt2", model),
         model = as.factor(model)) %>%
  mutate(train_size = case_when(str_starts(training, "bllip-lg") ~ 42,
                                str_starts(training, "bllip-md") ~ 15,
                                str_starts(training, "bllip-sm") ~ 5,
                                str_starts(training, "bllip-xs") ~ 1),
         
         # Training vocabulary usually covaries with the training corpus.
         # But BPE models share a vocabulary across training corpora.
         training_vocab=as.factor(ifelse(str_detect(training, "gptbpe"), "gptbpe", as.character(training))),
         training_source=as.factor(str_replace(as.character(training), "-gptbpe", ""))
         ) %>%
  mutate(seed = as.factor(seed)) %>%
  select(-pid, -test_loss) %>%
  distinct(model, training, seed, .keep_all = TRUE)
table(language_model_data$seed)

         0        111        120        922       1111       3602       4301       7245       7877      28066      28068 
         4          7          6          5          4          1          1          1          1          1          1 
     44862      51272      64924 1581807512 1581807578 1581861474 1581955288 1582126320 1586986276 1587139950 1611262494 
         1          1          1          1          1          2          2          2          1          1          1 
1611265210 
         1 
table(model_deltas$seed)

       111        120        607        922       1111       3602       4301       7245       7877      28066      28068 
         9          9          1          9         14          3          3          3          3          3          3 
     44862      51272      64924 1581807512 1581807578 1581861474 1581955288 1582126320 1586986276 1587139950 1611262494 
         3          3          3          3          3          3          3          2          3          3          3 
1611265210 
         3 

First join delta-metric data with model auxiliary data.

model_deltas = model_deltas %>%
  merge(language_model_data, by = c("seed", "training", "model"), all=T) %>%
  drop_na()

model_deltas

Also join on the original linear model data, rather than collapsing to delta-metrics. This will support regressions later on that don’t collapse across folds.

Final data preprocessing

# Exclude ordered-neurons from all analyses.
model_deltas <- model_deltas %>%
  filter(model != "ordered-neurons")

# Exclude bad GPT models.
model_deltas <- model_deltas %>%
  filter(model != "gpt2" | !(seed %in% c(1581955288, 1581861474, 1582126320)))

Visualizations

The basics

all_data %>% ggplot(aes(x=corpus)) + geom_bar()

print(all_data %>% group_by(corpus) %>% summarise(n=n()))
all_data %>% 
  ggplot(aes(x=freq, color=corpus)) + geom_density()

all_data %>% 
  ggplot(aes(x=len, color=corpus)) + geom_density()

all_data %>% 
  ggplot(aes(x=surprisal, color=corpus)) + geom_density()

Predictive power and SG

model_deltas %>%
  ggplot(aes(x=sg_score, y=delta_test_mean)) +
    geom_errorbar(aes(ymin=delta_test_mean-delta_test_sem, ymax=delta_test_mean+delta_test_sem)) +
    geom_smooth(method="lm", se=T) +
    geom_point(stat="identity", position="dodge", alpha=1, size=3, aes(color=training_vocab, shape=model)) +
    ylab(metric) +
    xlab("Syntax Generalization Score") +
    ggtitle("Syntactic Generalization vs. Predictive Power") +
    scale_color_manual(values = c("bllip-lg"="#440154FF",
                              "bllip-md"="#39568CFF",
                              "bllip-sm"="#1F968BFF",
                              "bllip-xs"="#73D055FF",
                              "gptbpe"="#888888")) +
    facet_grid(~corpus, scales="free") +
    theme(axis.text=element_text(size=14),
          strip.text.x = element_text(size=14),
          legend.text=element_text(size=14),
          axis.title=element_text(size=18),
          legend.position = "bottom")

#ggsave("./cogsci_images/sg_loglik.png",height=5,width=6)

Regression analyses

We control for effects of perplexity by relating the residuals of a performance ~ PPL regression to SG score.

# Prepare a residualized regression for x1 onto y, controlling for the effects of x2.
d_resid = model_deltas %>%
  drop_na() %>%
  
  group_by(corpus) %>%
    # Residualize delta metric w.r.t PPL for each model--training--seed within
    # training vocabulary
    mutate(resid.delta = resid(lm(delta_test_mean ~ training_vocab:test_ppl))) %>%
    # Residualize SG score w.r.t. PPL for each model--training--seed
    # within training vocabulary
    mutate(resid.sg = resid(lm(sg_score ~ training_vocab:test_ppl))) %>%
  ungroup()


# Now plot residual vs SG
d_resid %>%
  ggplot(aes(x=resid.sg, y=resid.delta)) +
    theme_bw() +
    scale_shape_manual(values = c(21, 24, 22, 23)) +
    geom_smooth(method="lm", se=T, alpha=0.3) +
    geom_point(stat="identity", position="dodge", alpha=1, size=5, aes(shape = model, fill=training_source, color = training_vocab, stroke = 1)) +
    ylab(paste("Residual", metric)) +
    xlab("Residual Syntax Generalization Score") +
    ggtitle("Syntactic Generalization vs. Predictive Power") +
    labs(color="training") + 
    scale_color_manual(values = c("bllip-lg"="#440154FF",
                                  "bllip-md"="#39568CFF",
                                  "bllip-sm"="#1F968BFF",
                                  "bllip-xs"="#73D055FF",
                                  "gptbpe"="#f0941f")) +
  scale_fill_manual(values = c("bllip-lg"="#440154FF",
                                  "bllip-md"="#39568CFF",
                                  "bllip-sm"="#1F968BFF",
                                  "bllip-xs"="#73D055FF",
                                  "gptbpe"="#f0941f"), guide=F) +
    facet_grid(.~corpus, scales="free") +
    theme(axis.text=element_text(size=14),
          strip.text.x = element_text(size=14),
          legend.text=element_text(size=14),
          axis.title=element_text(size=18),
          legend.position = "right")
ggsave("../images/cogsci2020/dll_sg.pdf",height=4.5,width=9, device = cairo_pdf)

do_stepwise_regression = function(cur_corpus) {
  regression_data = model_deltas %>%
    filter(corpus == cur_corpus)
  
  print("----------------------")
  print(cur_corpus)
  
  lm1 = lm(delta_test_mean ~ training_vocab:test_ppl, data = regression_data)
  lm2 = lm(delta_test_mean ~ training_vocab:test_ppl + sg_score, data = regression_data)
  print(anova(lm1, lm2))
  summary(lm2)
}
do_stepwise_regression("bnc-brown")
[1] "----------------------"
[1] "bnc-brown"
Analysis of Variance Table

Model 1: delta_test_mean ~ training_vocab:test_ppl
Model 2: delta_test_mean ~ training_vocab:test_ppl + sg_score
  Res.Df        RSS Df  Sum of Sq      F  Pr(>F)  
1     22 1.0876e-04                               
2     21 8.9322e-05  1 1.9435e-05 4.5692 0.04447 *
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Call:
lm(formula = delta_test_mean ~ training_vocab:test_ppl + sg_score, 
    data = regression_data)

Residuals:
       Min         1Q     Median         3Q        Max 
-0.0029317 -0.0011424 -0.0000891  0.0008578  0.0045715 

Coefficients:
                                  Estimate Std. Error t value Pr(>|t|)    
(Intercept)                      1.617e-02  1.936e-03   8.355 4.08e-08 ***
sg_score                        -7.830e-03  3.663e-03  -2.138  0.04447 *  
training_vocabbllip-lg:test_ppl  3.828e-05  1.944e-05   1.969  0.06232 .  
training_vocabbllip-md:test_ppl -2.084e-05  1.631e-05  -1.278  0.21513    
training_vocabbllip-sm:test_ppl -4.736e-05  1.387e-05  -3.415  0.00260 ** 
training_vocabbllip-xs:test_ppl -5.460e-05  9.157e-06  -5.963 6.43e-06 ***
training_vocabgptbpe:test_ppl   -1.472e-05  4.832e-06  -3.046  0.00614 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.002062 on 21 degrees of freedom
Multiple R-squared:  0.7944,    Adjusted R-squared:  0.7356 
F-statistic: 13.52 on 6 and 21 DF,  p-value: 2.906e-06
do_stepwise_regression("dundee")
[1] "----------------------"
[1] "dundee"
Analysis of Variance Table

Model 1: delta_test_mean ~ training_vocab:test_ppl
Model 2: delta_test_mean ~ training_vocab:test_ppl + sg_score
  Res.Df        RSS Df  Sum of Sq      F    Pr(>F)    
1     22 4.9377e-05                                   
2     21 1.9840e-05  1 2.9537e-05 31.263 1.505e-05 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Call:
lm(formula = delta_test_mean ~ training_vocab:test_ppl + sg_score, 
    data = regression_data)

Residuals:
       Min         1Q     Median         3Q        Max 
-1.454e-03 -5.196e-04 -7.965e-05  3.688e-04  1.774e-03 

Coefficients:
                                  Estimate Std. Error t value Pr(>|t|)    
(Intercept)                      1.029e-02  9.124e-04  11.276 2.27e-10 ***
sg_score                        -9.652e-03  1.726e-03  -5.591 1.51e-05 ***
training_vocabbllip-lg:test_ppl  2.365e-06  9.163e-06   0.258 0.798810    
training_vocabbllip-md:test_ppl -1.215e-06  7.685e-06  -0.158 0.875920    
training_vocabbllip-sm:test_ppl -1.600e-05  6.536e-06  -2.448 0.023220 *  
training_vocabbllip-xs:test_ppl -1.757e-05  4.316e-06  -4.071 0.000548 ***
training_vocabgptbpe:test_ppl   -3.087e-06  2.277e-06  -1.356 0.189571    
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.000972 on 21 degrees of freedom
Multiple R-squared:  0.719, Adjusted R-squared:  0.6387 
F-statistic: 8.955 on 6 and 21 DF,  p-value: 6.472e-05
do_stepwise_regression("natural-stories")
[1] "----------------------"
[1] "natural-stories"
Analysis of Variance Table

Model 1: delta_test_mean ~ training_vocab:test_ppl
Model 2: delta_test_mean ~ training_vocab:test_ppl + sg_score
  Res.Df        RSS Df  Sum of Sq      F  Pr(>F)  
1     22 3.8365e-05                               
2     21 2.8800e-05  1 9.5652e-06 6.9747 0.01528 *
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Call:
lm(formula = delta_test_mean ~ training_vocab:test_ppl + sg_score, 
    data = regression_data)

Residuals:
       Min         1Q     Median         3Q        Max 
-0.0017542 -0.0007293  0.0001275  0.0005964  0.0020843 

Coefficients:
                                  Estimate Std. Error t value Pr(>|t|)    
(Intercept)                      1.142e-02  1.099e-03  10.391 9.85e-10 ***
sg_score                        -5.493e-03  2.080e-03  -2.641  0.01528 *  
training_vocabbllip-lg:test_ppl -5.622e-05  1.104e-05  -5.092 4.82e-05 ***
training_vocabbllip-md:test_ppl -4.904e-05  9.259e-06  -5.297 2.99e-05 ***
training_vocabbllip-sm:test_ppl -6.074e-05  7.875e-06  -7.713 1.47e-07 ***
training_vocabbllip-xs:test_ppl -3.762e-05  5.200e-06  -7.235 3.97e-07 ***
training_vocabgptbpe:test_ppl   -8.025e-06  2.744e-06  -2.925  0.00809 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.001171 on 21 degrees of freedom
Multiple R-squared:  0.7816,    Adjusted R-squared:  0.7192 
F-statistic: 12.53 on 6 and 21 DF,  p-value: 5.315e-06
# The residualized analysis and the stepwise regression analysis
# should yield the same coefficients for the SG score variable.
#
# Below, we compute the slope coefficient for the SG term in the
# residualized analyses.
#
# These coefficients should match those found in the stepwise
# regression for `sg_score` above.
d_resid %>% group_by(corpus) %>%
  group_modify(~tidy(lm(resid.delta ~ training_vocab:test_ppl + resid.sg, data=.))
                 %>% filter(term == "resid.sg")) %>% 
  select(corpus, estimate)

Predictive power and perplexity


model_deltas %>%
  mutate(test_ppl = if_else(test_ppl > 500, 329.9, test_ppl),
         bpe = if_else(training_vocab == "gptbpe", "yes", "no")) %>%
  ggplot(aes(x=test_ppl, y=delta_test_mean, shape = model, ymin=0)) +
    theme_bw() +
    geom_text(aes(x=275, y=0, label = c("//"))) +
    geom_errorbar(aes(ymin=delta_test_mean-delta_test_sem, ymax=delta_test_mean+delta_test_sem, color=training_vocab), alpha=0.4) +
    #geom_smooth(method="lm", se=F) +
    geom_point(stat="identity", position="dodge", alpha=1, size=5, aes(fill=training_source, color = training_vocab, stroke = 1)) +
    ylab("ΔLogLik per token") +
    xlab("Test Perplexity") +
    #coord_cartesian(ylim = c(1, 16)) +
    ggtitle("Test Perplexity vs. Predictive Power") +
    labs(color="training") + 
    scale_color_manual(values = c("bllip-lg"="#440154FF",
                                  "bllip-md"="#39568CFF",
                                  "bllip-sm"="#1F968BFF",
                                  "bllip-xs"="#73D055FF",
                                  "gptbpe"="#f0941f")) +
  scale_fill_manual(values = c("bllip-lg"="#440154FF",
                                  "bllip-md"="#39568CFF",
                                  "bllip-sm"="#1F968BFF",
                                  "bllip-xs"="#73D055FF",
                                  "gptbpe"="#f0941f"), guide=F) +
    scale_shape_manual(values = c(21, 24, 22, 23)) +
    scale_x_continuous(labels=c(0, 50, 100, 150, 200, 250, 500 ,550), breaks=c(0, 50, 100, 150, 200, 250, 300, 350), minor_breaks = NULL) +
    scale_y_continuous(limits = c(0, NA), expand = c(0,0)) +
    facet_wrap(~corpus, scales="free") +
    coord_cartesian(clip="off") +
    theme(axis.text=element_text(size=12),
          strip.text.x = element_text(size=12),
          legend.text=element_text(size=12),
          axis.title=element_text(size=12),
          legend.position = "right")
ggsave("../images/cogsci2020/ppl_loglik.pdf",height=5,width=12, device = cairo_pdf)

Regression: Impact of PPL on Predictive Power

lmd = model_deltas %>%
  mutate(training_vocab=ifelse(str_detect(as.character(training), "gptbpe"),
                               "gptbpe", as.character(training)))
summary(lmer(delta_test_mean ~ training_vocab:test_ppl + (1 | corpus) + (1 | model), data=lmd))
Linear mixed model fit by REML. t-tests use Satterthwaite's method ['lmerModLmerTest']
Formula: delta_test_mean ~ training_vocab:test_ppl + (1 | corpus) + (1 |      model)
   Data: lmd

REML criterion at convergence: -675.9

Scaled residuals: 
    Min      1Q  Median      3Q     Max 
-2.3376 -0.5096 -0.1373  0.5837  2.9157 

Random effects:
 Groups   Name        Variance  Std.Dev.
 model    (Intercept) 2.271e-06 0.001507
 corpus   (Intercept) 6.634e-06 0.002576
 Residual             3.728e-06 0.001931
Number of obs: 84, groups:  model, 4; corpus, 3

Fixed effects:
                                  Estimate Std. Error         df t value Pr(>|t|)   
(Intercept)                      9.383e-03  1.976e-03  5.167e+00   4.748  0.00470 **
training_vocabbllip-lg:test_ppl -2.150e-06  1.745e-05  2.162e+01  -0.123  0.90309   
training_vocabbllip-md:test_ppl -2.065e-05  1.484e-05  2.109e+01  -1.391  0.17858   
training_vocabbllip-sm:test_ppl -3.100e-05  1.286e-05  1.923e+01  -2.410  0.02611 * 
training_vocabbllip-xs:test_ppl -2.801e-05  8.655e-06  1.885e+01  -3.236  0.00438 **
training_vocabgptbpe:test_ppl   -9.676e-06  2.825e-06  7.472e+01  -3.424  0.00100 **
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Correlation of Fixed Effects:
                  (Intr) trnng_vcbbllp-l:_ trnng_vcbbllp-m:_ trnng_vcbbllp-s:_ trnng_vcbbllp-x:_
trnng_vcbbllp-l:_ -0.482                                                                        
trnng_vcbbllp-m:_ -0.494  0.828                                                                 
trnng_vcbbllp-s:_ -0.493  0.829             0.851                                               
trnng_vcbbllp-x:_ -0.500  0.840             0.864             0.871                             
trnng_vcb:_       -0.299  0.477             0.487             0.454             0.461           

Perplexity vs. SG Score

This is a reproduction of Figure 2 from Hu et al.


model_deltas %>%
  mutate(test_ppl = if_else(test_ppl > 500, 329.9, test_ppl)) %>%
  mutate(train_size = log(train_size)) %>%
  mutate(bpe = if_else(training_vocab == "gptbpe", "yes", "no"),
         bpe = as.factor(bpe)) %>%
  ggplot(aes(x=test_ppl, y=sg_score)) +
    theme_bw() +
    geom_hline(yintercept = 0.28, linetype = "dashed", color="gray") +
    geom_text(aes(x=240, y=0.3), label="random", color="gray") +
    geom_point(stat="identity", position="dodge", alpha=0.3, size=4, aes(shape = model, fill=training_source, color = training_vocab, stroke = 1)) +
    geom_text(aes(x=275, y=0, label = c("//"))) +
    ylab("SG Score") +
    xlab("Test Perplexity") +
    ggtitle("Test PPL vs. SG Score") +
    labs(color="training") + 
    scale_color_manual(values = c("bllip-lg"="#440154FF",
                                  "bllip-md"="#39568CFF",
                                  "bllip-sm"="#1F968BFF",
                                  "bllip-xs"="#73D055FF",
                                  "gptbpe"="#f0941f")) +
    scale_fill_manual(values = c("bllip-lg"="#440154FF",
                                  "bllip-md"="#39568CFF",
                                  "bllip-sm"="#1F968BFF",
                                  "bllip-xs"="#73D055FF",
                                  "gptbpe"="#f0941f"), guide=F) +
    scale_shape_manual(values = c("5gram"=21, vanilla=22, gpt2=24, rnng=23)) +
    scale_x_continuous(labels=c(0, 50, 100, 150, 200, 250, 500 ,550), breaks=c(0, 50, 100, 150, 200, 250, 300, 350), minor_breaks = NULL) +
    scale_y_continuous(limits = c(0, 1), expand = c(0,0)) +
    theme(axis.text=element_text(size=12),
          strip.text.x = element_text(size=12),
          legend.text=element_text(size=8),
          legend.title=element_text(size=8),
          axis.title=element_text(size=14),
          legend.position = "none",
          legend.direction = "horizontal",
          legend.key.width = unit(0.3,"cm"),
          legend.spacing.x = unit(0.1, 'cm'))
ggsave("../images/cogsci2020/ppl_sg.pdf",height=4.5,width=3, device = cairo_pdf)

Smith & Levy reproduction

This redone so that it’s unique for each model

all_data %>%
  ggplot(aes(x=surprisal, color=model)) +
  theme_bw() +
  geom_density() +
  facet_wrap(.~corpus, ncol=1, scales="free", strip.position = "right") +
  coord_cartesian(xlim = c(0, 25)) +
  ggtitle("Distribution of Surprisal") +
ggsave("../images/cogsci2020/surp_corr_marginals.png",height=5,width=4)
k = 1.97

# Fit a GAM for a bootstrap sample.
fit_gam_inner = function(bootstrap_sample, key) {
  # This bootstrap sample may have repeated elements. That causes a problem for
  # mgcv, which internally cross-validates some model parameters -- it may
  # allocate repeated elements to different folds and thus double-dip. We'll
  # prevent this by instead providing the whole (pre-bootstrap) dataset to mgcv,
  # and using `weights` to constrain which elements are seen, and how many
  # times. (Repeated elements of the sample may get a weight of 2 or 3 or N,
  # which is exactly what we want.)
  
  # rsplit$data contains the original entire dataset.
  df = bootstrap_sample$data
  # as.integer.rsplit returns the indices of the examples which are in-sample.
  # convert this to a count vector, with dimension N (total dataset rows)
  weights = tabulate(as.integer(bootstrap_sample), nrow(df))
  
  if (key$corpus == "dundee") {
    # Reading time regression: use features of current and previous word
    m = gam(psychometric ~ s(surprisal, bs = 'cr', k = 20) + s(prev_surp, bs = 'cr', k = 20) +
                           te(freq, len, bs = 'cr') + te(prev_freq, prev_len, bs = 'cr'),
            data = df, weights = weights)
    
    terms_to_predict = c("s(surprisal)", "s(prev_surp)")
  } else {
    # SPRT regression: use features of current and 3 previous words
    m = gam(psychometric ~ s(surprisal, bs = 'cr', k = 20) + s(prev_surp, bs = 'cr', k = 20) +
                           s(prev2_surp, bs = 'cr', k = 20) + s(prev3_surp, bs = 'cr', k = 20) +
                           te(freq, len, bs = 'cr') + te(prev_freq, prev_len, bs = 'cr') +
                           te(prev2_freq, prev2_len, bs = 'cr') + te(prev3_freq, prev3_len, bs = 'cr'),
            data = df, weights = weights)
    
    terms_to_predict = c("s(surprisal)", "s(prev_surp)",
                         "s(prev2_surp)", "s(prev3_surp)")
  }

  # Produce psychometric predictions line using just the relevant context-specific predictors.

  newdata = data.frame(surprisal=seq(0,20,by=0.1),
                       prev_surp=seq(0,20,by=0.1),
                       prev2_surp=seq(0,20,by=0.1),
                       prev3_surp=seq(0,20,by=0.1),
                       freq=0, prev_freq=0, prev2_freq=0, prev3_freq=0,
                       len=0, prev_len=0, prev2_len=0, prev3_len=0)
  
  # Returns a matrix N_samples * N_terms.
  per_term_predictions = predict(m, newdata=newdata, terms=terms_to_predict, type="terms")
  
  # Additive model -- sum across predictor response contributions (matrix columns).
  predictions = rowSums(per_term_predictions)
  
  return(newdata %>% mutate(y=predictions))
}

# Fit a bootstrap-re-estimated GAM for the given model--corpus--training group.
fit_gam = function(df, key, alpha=0.05) {
  # Bootstrap-resample data
  boot_models = df %>% bootstraps(times=50) %>% 
    # Fit a GAM and get predictions for each sample
    mutate(smoothed=map(splits, fit_gam_inner, key=key))
  
  # Extract mean and 5% and 95% percentile y-values for each surprisal value
  result = boot_models %>% 
    unnest(smoothed) %>% 
    select(surprisal, y) %>% 
    group_by(surprisal) %>% 
      summarise(y_lower=quantile(y, alpha / 2), 
                y_upper=quantile(y, 1 - alpha / 2),
                y=mean(y)) %>% 
    ungroup()
  
  return (result)
}

smooths = all_data %>%
  mutate(
    training_vocab=as.factor(ifelse(str_detect(training, "gptbpe"), "gptbpe", as.character(training))),
    training_source=as.factor(str_replace(as.character(training), "-gptbpe", ""))) %>%
  group_by(training_vocab, training_source, model, corpus) %>%
    group_modify(fit_gam) %>%
  ungroup()
write.csv(smooths, "../data/gam_smooths.csv")

Plot the GAM model fits

ymin = -40
ymax = 100
xmin = 0
xmax = 20

get_d_points = function(df, model, training, corpus){
  x = density(df$surprisal)$x
  y = density(df$surprisal)$y
  return(data.frame(model, training, corpus, x, y))
}

# Get the density points
density_data = all_data %>%
  mutate(model = recode(model, vanilla="lstm")) %>%
  group_by(model, training, corpus) %>%
    do({get_d_points(., unique(.$model), unique(.$training), unique(.$corpus))}) %>%
  ungroup() %>%
  mutate(training_vocab=as.factor(ifelse(str_detect(training, "gptbpe"), "gptbpe", as.character(training))),
         training_source=as.factor(str_replace(as.character(training), "-gptbpe", "")),
         bpe=training_vocab == "gptbpe",) %>% 
  filter(x>0, x<20)

smooths = read.csv("../data/gam_smooths.sl2013.all.csv")
cannot open file '../data/gam_smooths.sl2013.all.csv': No such file or directoryError in file(file, "rt") : cannot open the connection
LS0tCnRpdGxlOiAiQ29nU2NpIDIwMjAgQW5hbHlzaXMiCm91dHB1dDogaHRtbF9ub3RlYm9vawotLS0KCiMgUGFja2FnZXMgYW5kIHV0aWxpdGllcwoKYGBge3J9CmxpYnJhcnkodGlkeXZlcnNlKQpsaWJyYXJ5KGxtZTQpCmxpYnJhcnkobG1lclRlc3QpCmxpYnJhcnkobG9nZ2luZykKbGlicmFyeShtdnRub3JtKQpsaWJyYXJ5KG1nY3YpCiMgUHJvdmlkZXMgYm9vdHN0cmFwIHJlc2FtcGxpbmcgdG9vbHMKbGlicmFyeShyc2FtcGxlKQpgYGAKCmBgYHtyfQojIENvbXB1dGUgdGhlIGxvZy1saWtlbGlob29kIG9mIGEgbmV3IGRhdGFzZXQgdXNpbmcgYSBmaXQgbG1lNCBtb2RlbC4KbG9nTGlrX3Rlc3QgPC0gZnVuY3Rpb24obG0sIHRlc3RfWCwgdGVzdF95KSB7CiAgcHJlZGljdGlvbnMgPC0gcHJlZGljdChsbSwgdGVzdF9YLCByZS5mb3JtPU5BKQogICMgR2V0IHN0ZC5kZXYuIG9mIHJlc2lkdWFsLCBlc3RpbWF0ZWQgZnJvbSB0cmFpbiBkYXRhCiAgc3RkZXYgPC0gc2lnbWEobG0pCiAgIyBGb3IgZWFjaCBwcmVkaWN0aW9uLS1vYnNlcnZhdGlvbiwgZ2V0IHRoZSBkZW5zaXR5IHAob2JzIHwgTihwcmVkaWN0ZWQsIG1vZGVsX3NpZ21hKSkgYW5kIHJlZHVjZQogIGRlbnNpdHkgPC0gc3VtKGRub3JtKHRlc3RfeSwgcHJlZGljdGlvbnMsIHN0ZGV2LCBsb2c9VFJVRSkpCiAgcmV0dXJuKGRlbnNpdHkpCn0KIyBHZXQgcGVyLXByZWRpY3Rpb24gbG9nLWxpa2VsaWhvb2QKbG9nTGlrX3Rlc3RfcGVyIDwtIGZ1bmN0aW9uKGxtLCB0ZXN0X1gsIHRlc3RfeSkgewogIHByZWRpY3Rpb25zIDwtIHByZWRpY3QobG0sIHRlc3RfWCwgcmUuZm9ybT1OQSkKICAjIEdldCBzdGQuZGV2LiBvZiByZXNpZHVhbCwgZXN0aW1hdGVkIGZyb20gdHJhaW4gZGF0YQogIHN0ZGV2IDwtIHNpZ21hKGxtKQogICMgRm9yIGVhY2ggcHJlZGljdGlvbi0tb2JzZXJ2YXRpb24sIGdldCB0aGUgZGVuc2l0eSBwKG9icyB8IE4ocHJlZGljdGVkLCBtb2RlbF9zaWdtYSkpCiAgZGVuc2l0aWVzIDwtIGRub3JtKHRlc3RfeSwgcHJlZGljdGlvbnMsIHN0ZGV2LCBsb2c9VFJVRSkKICByZXR1cm4oZGVuc2l0aWVzKQp9CiMgQ29tcHV0ZSBNU0Ugb2YgYSBuZXcgZGF0YXNldCB1c2luZyBhIGZpdCBsbWU0IG1vZGVsLgptc2VfdGVzdCA8LSBmdW5jdGlvbihsbSwgdGVzdF9YLCB0ZXN0X3kpIHsKICByZXR1cm4obWVhbigocHJlZGljdChsbSwgdGVzdF9YLCByZS5mb3JtPU5BKSAtIHRlc3RfeSkgXiAyKSkKfQojU2FuaXR5IGNoZWNrcwojbXlsbSA8LSBnYW0ocHN5Y2hvbWV0cmljIH4gIHMoc3VycHJpc2FsLCBicyA9ICJjciIsIGsgPSAyMCkgKyBzKHByZXZfc3VycCwgYnMgPSAiY3IiLCBrID0gMjApICsgdGUoZnJlcSwgbGVuLCBicyA9ICJjciIpICsgdGUocHJldl9mcmVxLCBwcmV2X2xlbiwgYnMgPSAiY3IiKSwgZGF0YT10cmFpbl9kYXRhKQojYyhsb2dMaWsobXlsbSksIGxvZ0xpa190ZXN0KG15bG0sIHRyYWluX2RhdGEsIHRyYWluX2RhdGEkcHN5Y2hvbWV0cmljKSkKI2xvZ0xpa190ZXN0KG15bG0sIHRlc3RfZGF0YSwgdGVzdF9kYXRhJHBzeWNob21ldHJpYykKYGBgCgojIERhdGEgbG9hZGluZyBhbmQgcHJlcHJvY2Vzc2luZwoKYGBge3IgTG9hZCBhbmQgcHJlcHJvY2VzcyBkYXRhfQpkYXRhID0gcmVhZC5jc3YoIi4uL2RhdGEvaGFybW9uaXplZF9yZXN1bHRzLmNzdiIpCgphbGxfZGF0YSA9IGRhdGEgJT4lCiAgbXV0YXRlKHNlZWQgPSBhcy5mYWN0b3Ioc2VlZCkpICU+JQogIGdyb3VwX2J5KGNvcnB1cywgbW9kZWwsIHRyYWluaW5nLCBzZWVkKSAlPiUKICAgIG11dGF0ZShwcmV2X3N1cnAgPSBsYWcoc3VycHJpc2FsKSwKICAgICAgICAgcHJldl9jb2RlID0gbGFnKGNvZGUpLAogICAgICAgICBwcmV2X2xlbiA9IGxhZyhsZW4pLAogICAgICAgICBwcmV2X2ZyZXEgPSBsYWcoZnJlcSksCiAgICAgICAgIHByZXZfc3VycCA9IGxhZyhzdXJwcmlzYWwpLAogICAgICAgICAKICAgICAgICAgcHJldjJfZnJlcSA9IGxhZyhwcmV2X2ZyZXEpLAogICAgICAgICBwcmV2Ml9jb2RlID0gbGFnKHByZXZfY29kZSksCiAgICAgICAgIHByZXYyX2xlbiA9IGxhZyhwcmV2X2xlbiksCiAgICAgICAgIHByZXYyX3N1cnAgPSBsYWcocHJldl9zdXJwKSwKICAgICAgICAgCiAgICAgICAgIHByZXYzX2ZyZXEgPSBsYWcocHJldjJfZnJlcSksCiAgICAgICAgIHByZXYzX2NvZGUgPSBsYWcocHJldjJfY29kZSksCiAgICAgICAgIHByZXYzX2xlbiA9IGxhZyhwcmV2Ml9sZW4pLAogICAgICAgICBwcmV2M19zdXJwID0gbGFnKHByZXYyX3N1cnApLAogICAgICAgICAKICAgICAgICAgcHJldjRfZnJlcSA9IGxhZyhwcmV2M19mcmVxKSwKICAgICAgICAgcHJldjRfY29kZSA9IGxhZyhwcmV2M19jb2RlKSwKICAgICAgICAgcHJldjRfbGVuID0gbGFnKHByZXYzX2xlbiksCiAgICAgICAgIHByZXY0X3N1cnAgPSBsYWcocHJldjNfc3VycCkpICU+JQogIHVuZ3JvdXAoKSAlPiUKICAKICAjIEZpbHRlciBiYWNrIHR3byBmb3IgdGhlIGR1bmRlZSBjb3JwdXMuIEZpbHRlciBiYWNrIDEgZm9yIGFsbCBvdGhlciBjb3Jwb3JhCiAgIyBOQiB0aGlzIGVmZmVjdGl2ZWx5IHJlbW92ZXMgYWxsIHplcm8tc3VycHJpc2FsIHJvd3MsIHNpbmNlIGVhcmx5LXNlbnRlbmNlIHRva2VucyBkb24ndCBoYXZlIGNvbnRpZ3VvdXMgdG9rZW4gaGlzdG9yeQogIGZpbHRlcigoY29ycHVzID09ICJkdW5kZWUiICYgY29kZSA9PSBwcmV2Ml9jb2RlICsgMikgfCAoY29ycHVzICE9ICJkdW5kZWUiICYgY29kZSA9PSBwcmV2NF9jb2RlICsgNCkpICU+JQogIAogIHNlbGVjdCgtcHJldl9jb2RlLCAtcHJldjJfY29kZSwgLXByZXYzX2NvZGUpICU+JQogIGRyb3BfbmEoKQoKYWxsX2RhdGEgPSBhbGxfZGF0YSAlPiUKICBtdXRhdGUoCiAgICBtb2RlbCA9IGFzLmNoYXJhY3Rlcihtb2RlbCksCiAgICBtb2RlbCA9IGlmX2Vsc2UobW9kZWwgPT0gImdwdC0yIiwgImdwdDIiLCBtb2RlbCksCiAgICBtb2RlbCA9IGFzLmZhY3Rvcihtb2RlbCkpCmBgYAoKYGBge3J9Cm1pc3Npbmdfcm93cyA9IGFsbF9kYXRhICU+JSBjb21wbGV0ZShuZXN0aW5nKGNvcnB1cywgY29kZSksIG5lc3RpbmcobW9kZWwsIHRyYWluaW5nLCBzZWVkKSkgJT4lIAogIGdyb3VwX2J5KGNvcnB1cywgY29kZSkgJT4lIAogICAgZmlsdGVyKHN1bShpcy5uYShzdXJwcmlzYWwpKSA+IDApICU+JSAKICB1bmdyb3VwKCkgJT4lIAogIGFudGlfam9pbihhbGxfZGF0YSwgYnk9YygiY29ycHVzIiwgImNvZGUiLCAibW9kZWwiLCAidHJhaW5pbmciLCAic2VlZCIpKQoKbWlzc2luZ19yb3dzICU+JSBnZ3Bsb3QoYWVzKHg9Y29ycHVzLCBmaWxsPWZhY3RvcihwYXN0ZShtb2RlbCx0cmFpbmluZykpKSkgKwpnZW9tX2Jhcihwb3NpdGlvbj1wb3NpdGlvbl9kb2RnZSh3aWR0aD0wLjgpKQpwcmludChtaXNzaW5nX3Jvd3MgJT4lIGdyb3VwX2J5KG1vZGVsLCB0cmFpbmluZywgc2VlZCwgY29ycHVzKSAlPiUgc3VtbWFyaXNlKG49bigpKSAlPiUgYXJyYW5nZShkZXNjKG4pKSkKYGBgCgoKYGBge3IgRHJvcCB0b2tlbnMgZm9yIHdoaWNoIGFueSBtb2RlbCBpcyBtaXNzaW5nIHN1cnByaXNhbCBkYXRhLn0KCiMgQ29tcHV0ZSB0aGUgaWRlYWwgbnVtYmVyIG9mIG1vZGVsLS1zZWVkLS10cmFpbmluZyBvYnNlcnZhdGlvbnMgcGVyIHRva2VuLgp0b19kcm9wID0gYWxsX2RhdGEgJT4lCiAgZ3JvdXBfYnkoY29ycHVzLCBjb2RlKSAlPiUgc3VtbWFyaXNlKG4gPSBuKCkpICU+JSB1bmdyb3VwKCkgJT4lCiAgZ3JvdXBfYnkoY29ycHVzKSAlPiUgbXV0YXRlKCBtYXhfbiA9IG1heChuKSkgJT4lIHVuZ3JvdXAoKSAlPiUKICBmaWx0ZXIobWF4X24gIT0gbikgJT4lIAogIHNlbGVjdChjb2RlLCBjb3JwdXMpCgojIEZpbmQgdG9rZW5zIHdoaWNoIGhhdmUgMjggb2JzZXJ2YXRpb25zIGFuZCBjb21wYXJlIG1vZGVsK3RyYWluaW5nIGZyZXFzCmFsbF9kYXRhICU+JSBmaWx0ZXIoY29ycHVzID09ICJibmMtYnJvd24iKSAlPiUgZ3JvdXBfYnkoY29kZSkgJT4lIGZpbHRlcihuKCkgPT0gMjgpCnRlbXB4ID0gYWxsX2RhdGEgJT4lIGZpbHRlcihjb3JwdXMgPT0gImJuYy1icm93biIsIGNvZGUgPT0gMTcxMDMpCnRhYmxlKHBhc3RlKHRlbXB4JG1vZGVsLCB0ZW1weCR0cmFpbmluZykpCgojIFRoaXMgb25lIGlzIG1pc3NpbmcgYSBzZWVkCiN0b19kcm9wICU+JSBmaWx0ZXIoY29ycHVzID09ICJibmMtYnJvd24iKSAlPiUgYXJyYW5nZShjb2RlKQp0ZW1weCA9IGFsbF9kYXRhICU+JSBmaWx0ZXIoY29ycHVzID09ICJibmMtYnJvd24iLCBjb2RlID09IDE3MDE3KQp0YWJsZShwYXN0ZSh0ZW1weCRtb2RlbCwgdGVtcHgkdHJhaW5pbmcpKQoKIyBudm0gc29tZWhvdyBub3QgYSBwcm9ibGVtIGFueW1vcmUgLi4KIyAjIFpvb21pbmcgaW4gb24gdGhlIHByb2JsZW0gLS0gd2h5IGlzIHRoZXJlIG5vIGJsbGlwLWxnIGRhdGEgaGVyZT8KIyBhbGxfZGF0YSAlPiUgZmlsdGVyKGNvcnB1cyA9PSAiZHVuZGVlIiwgbW9kZWwgPT0gInZhbmlsbGEiLCB0cmFpbmluZyA9PSAiYmxsaXAtbGciLCBjb2RlID4gMTA3MjAsIGNvZGUgPCAxMDczMCkKCmxvZ2luZm8ocGFzdGUoIkRyb3BwaW5nIiwgbnJvdyh0b19kcm9wKSwgIm9ic2VydmF0aW9ucyBjb3JyZXNwb25kaW5nIHRvIGNvcnB1cyB0b2tlbnMgd2hpY2ggYXJlIG1pc3Npbmcgb2JzZXJ2YXRpb25zIGZvciBzb21lIG1vZGVsLiIpKQpsb2dpbmZvKHBhc3RlKCJEcm9wcGluZyIsIHRvX2Ryb3AgJT4lIGdyb3VwX2J5KGNvcnB1cywgY29kZSkgJT4lIG5fZ3JvdXBzKCksICJ0b2tlbnMgd2hpY2ggYXJlIG1pc3Npbmcgb2JzZXJ2YXRpb25zIGZvciBzb21lIG1vZGVsLiIpKQoKYWxsX2RhdGEgPSBhbGxfZGF0YSAlPiUgYW50aV9qb2luKHRvX2Ryb3AgJT4lIGdyb3VwX2J5KGNvcnB1cywgY29kZSksIGJ5PWMoImNvcnB1cyIsICJjb2RlIikpCmxvZ2luZm8ocGFzdGUoIkFmdGVyIGRyb3AsIiwgbnJvdyhhbGxfZGF0YSksICJvYnNlcnZhdGlvbnMgKCIsIGFsbF9kYXRhICU+JSBncm91cF9ieShjb3JwdXMsIGNvZGUpICU+JSBuX2dyb3VwcygpLCAiIHRva2VucykgcmVtYWluLiIpKQpgYGAKCmBgYHtyIERyb3AgdG9rZW5zIGZvciB3aGljaCBhbnkgbW9kZWwgaGFzIHplcm8tdmFsdWVkIHN1cnByaXNhbHMufQoKdG9fZHJvcF96ZXJvX3N1cnBzID0gYWxsX2RhdGEgJT4lIGdyb3VwX2J5KGNvcnB1cywgY29kZSkgJT4lIGZpbHRlcihhbnkoc3VycHJpc2FsID09IDApKSAlPiUgdW5ncm91cCgpCmxvZ2luZm8ocGFzdGUoIkRyb3BwaW5nIiwgbnJvdyh0b19kcm9wX3plcm9fc3VycHMpLCAib2JzZXJ2YXRpb25zIGNvcnJlc3BvbmRpbmcgdG8gY29ycHVzIHRva2VucyB3aGljaCBoYXZlIHN1cnByaXNhbCB6ZXJvcyBmb3Igc29tZSBtb2RlbC4iKSkKbG9naW5mbyhwYXN0ZSgiRHJvcHBpbmciLCB0b19kcm9wX3plcm9fc3VycHMgJT4lIGdyb3VwX2J5KGNvcnB1cywgY29kZSkgJT4lIG5fZ3JvdXBzKCksICJ0b2tlbnMgd2hpY2ggaGF2ZSBzdXJwcmlzYWwgemVyb3MgZm9yIHNvbWUgbW9kZWwuIikpCgphbGxfZGF0YSA9IGFsbF9kYXRhICU+JSBhbnRpX2pvaW4odG9fZHJvcF96ZXJvX3N1cnBzICU+JSBncm91cF9ieShjb3JwdXMsIGNvZGUpLCBieT1jKCJjb3JwdXMiLCAiY29kZSIpKQpsb2dpbmZvKHBhc3RlKCJBZnRlciBkcm9wLCIsIG5yb3coYWxsX2RhdGEpLCAib2JzZXJ2YXRpb25zICgiLCBhbGxfZGF0YSAlPiUgZ3JvdXBfYnkoY29ycHVzLCBjb2RlKSAlPiUgbl9ncm91cHMoKSwgIiB0b2tlbnMpIHJlbWFpbi4iKSkKYGBgCgpgYGB7ciBEcm9wIHRva2VucyBmb3Igd2hpY2ggd2UgaGF2ZSB6ZXJvLXZhbHVlZCBwc3ljaG9tZXRyaWMgZGF0YS59Cgp0b19kcm9wX3plcm9fcHN5Y2hzID0gYWxsX2RhdGEgJT4lIGdyb3VwX2J5KGNvcnB1cywgY29kZSkgJT4lIGZpbHRlcihhbnkocHN5Y2hvbWV0cmljID09IDApKSAlPiUgdW5ncm91cCgpCmxvZ2luZm8ocGFzdGUoIkRyb3BwaW5nIiwgbnJvdyh0b19kcm9wX3plcm9fcHN5Y2hzKSwgIm9ic2VydmF0aW9ucyBjb3JyZXNwb25kaW5nIHRvIGNvcnB1cyB0b2tlbnMgd2hpY2ggaGF2ZSBwc3ljaG9tZXRyaWMgemVyb3MgZm9yIHNvbWUgbW9kZWwuIikpCmxvZ2luZm8ocGFzdGUoIkRyb3BwaW5nIiwgdG9fZHJvcF96ZXJvX3BzeWNocyAlPiUgZ3JvdXBfYnkoY29ycHVzLCBjb2RlKSAlPiUgbl9ncm91cHMoKSwgInRva2VucyB3aGljaCBoYXZlIHBzeWNob21ldHJpYyB6ZXJvcyBmb3Igc29tZSBtb2RlbC4iKSkKCmFsbF9kYXRhID0gYWxsX2RhdGEgJT4lIGFudGlfam9pbih0b19kcm9wX3plcm9fcHN5Y2hzICU+JSBncm91cF9ieShjb3JwdXMsIGNvZGUpLCBieT1jKCJjb3JwdXMiLCAiY29kZSIpKQpsb2dpbmZvKHBhc3RlKCJBZnRlciBkcm9wLCIsIG5yb3coYWxsX2RhdGEpLCAib2JzZXJ2YXRpb25zICgiLCBhbGxfZGF0YSAlPiUgZ3JvdXBfYnkoY29ycHVzLCBjb2RlKSAlPiUgbl9ncm91cHMoKSwgIiB0b2tlbnMpIHJlbWFpbi4iKSkKYGBgCiAKIyBUcmFpbiBMaW5lYXIgTW9kZWxzIHdoaWNoIGFyZSB1c2VkIHRvIGFzc2VzcyBEZWx0YSBMb2cgTGlrCiAKYGBge3J9CiMgQ29tcHV0ZSBsaW5lYXIgbW9kZWwgc3RhdHMgZm9yIHRoZSBnaXZlbiB0cmFpbmluZyBkYXRhIHN1YnNldCBhbmQgZnVsbCB0ZXN0IGRhdGEuCiMgQXV0b21hdGljYWxseSBzdWJzZXRzIHRoZSB0ZXN0IGRhdGEgdG8gbWF0Y2ggdGhlIHJlbGV2YW50IGdyb3VwIGZvciB3aGljaCB3ZSBhcmUgdHJhaW5pbmcgYSBsaW5lYXIgbW9kZWwuCmdldF9sbV9kYXRhIDwtIGZ1bmN0aW9uKGRmLCB0ZXN0X2RhdGEsIGZvcm11bGEsIGZvbGQsIHN0b3JlX2VudikgewogICN0aGlzX2xtIDwtIGdhbShmb3JtdWxhLCBkYXRhPWRmKTsKICBwcmludChwYXN0ZSh1bmlxdWUocGFzdGUoZGYkbW9kZWwsIGRmJHRyYWluaW5nLCBkZiRzZWVkLCBkZiRjb3JwdXMpKVsxXSwgZm9sZCwgbnJvdyhkZikpKQogIHRoaXNfbG0gPSBsbShmb3JtdWxhLCBkYXRhPWRmKQogIHRoaXNfdGVzdF9kYXRhIDwtIHNlbWlfam9pbih0ZXN0X2RhdGEsIGRmLCBieT1jKCJ0cmFpbmluZyIsICJtb2RlbCIsICJzZWVkIiwgImNvcnB1cyIpKTsKICAKICAjIFNhdmUgbG0gdG8gdGhlIGdsb2JhbCBlbnYgc28gdGhhdCB3ZSBjYW4gYWNjZXNzIHJlc2lkdWFscyBsYXRlci4KICBsbV9uYW1lID0gcGFzdGUodW5pcXVlKHBhc3RlKGRmJG1vZGVsLCBkZiR0cmFpbmluZywgZGYkc2VlZCwgZGYkY29ycHVzKSlbMV0sIGZvbGQpCiAgYXNzaWduKGxtX25hbWUsIHRoaXNfbG0sIGVudmlyPXN0b3JlX2VudikKICAKICBzdW1tYXJpc2UoZGYsCiAgICAgICAgICAgIGxvZ19saWsgPSBhcy5udW1lcmljKGxvZ0xpayh0aGlzX2xtLCBSRU1MID0gRikpLAogICAgICAgICAgICB0ZXN0X2xpayA9IGxvZ0xpa190ZXN0KHRoaXNfbG0sIHRoaXNfdGVzdF9kYXRhLCB0aGlzX3Rlc3RfZGF0YSRwc3ljaG9tZXRyaWMpLAogICAgICAgICAgICB0ZXN0X21zZSA9IG1zZV90ZXN0KHRoaXNfbG0sIHRoaXNfdGVzdF9kYXRhLCB0aGlzX3Rlc3RfZGF0YSRwc3ljaG9tZXRyaWMpKQp9CiMgRm9yIGEgcHJldmlvdXNseSBmaXR0ZWQgbG0gc3RvcmVkIGluIHN0b3JlX2VudiwgZ2V0IHRoZSByZXNpZHVhbHMgb24gdGVzdCBkYXRhIG9mIHRoZSByZWxldmFudCBkYXRhIHN1YnNldC4KZ2V0X2xtX3Jlc2lkdWFscyA8LSBmdW5jdGlvbihkZiwgZm9sZCwgc3RvcmVfZW52KSB7CiAgIyBSZXRyaWV2ZSB0aGUgcmVsZXZhbnQgbG0uCiAgbG1fbmFtZSA9IHBhc3RlKHVuaXF1ZShwYXN0ZShkZiRtb2RlbCwgZGYkdHJhaW5pbmcsIGRmJHNlZWQsIGRmJGNvcnB1cykpWzFdLCBmb2xkKQogIHRoaXNfbG0gPC0gZ2V0KGxtX25hbWUsIGVudmlyPXN0b3JlX2VudikKICAKICBtdXRhdGUoZGYsCiAgICAgICAgIGxpa2VsaWhvb2QgPSBsb2dMaWtfdGVzdF9wZXIodGhpc19sbSwgZGYsIGRmJHBzeWNob21ldHJpYyksCiAgICAgICAgIHJlc2lkID0gZGYkcHN5Y2hvbWV0cmljIC0gcHJlZGljdCh0aGlzX2xtLCBkZiwgcmUuZm9ybT1OQSkpCn0KIyBDb21wdXRlIHBlci1leGFtcGxlIGRlbHRhLWxvZy1saWtlbGlob29kIGZvciB0aGUgZ2l2ZW4gdGVzdCBmb2xkLgpnZXRfbG1fZGVsdGFfbG9nX2xpayA8LSBmdW5jdGlvbih0ZXN0X2RhdGEsIGZvbGQsIGJhc2VsaW5lX2VudiwgZnVsbF9lbnYpIHsKICBsbV9uYW1lID0gcGFzdGUodW5pcXVlKHBhc3RlKHRlc3RfZGF0YSRtb2RlbCwgdGVzdF9kYXRhJHRyYWluaW5nLCB0ZXN0X2RhdGEkc2VlZCwgdGVzdF9kYXRhJGNvcnB1cykpWzFdLCBmb2xkKQogIGJhc2VsaW5lX2xtIDwtIGdldChsbV9uYW1lLCBlbnZpcj1iYXNlbGluZV9lbnYpCiAgZnVsbF9sbSA8LSBnZXQobG1fbmFtZSwgZW52aXI9ZnVsbF9lbnYpCiAgCiAgZGVsdGFfbG9nX2xpayA9IGxvZ0xpa190ZXN0X3BlcihmdWxsX2xtLCB0ZXN0X2RhdGEsIHRlc3RfZGF0YSRwc3ljaG9tZXRyaWMpIC0gbG9nTGlrX3Rlc3RfcGVyKGJhc2VsaW5lX2xtLCB0ZXN0X2RhdGEsIHRlc3RfZGF0YSRwc3ljaG9tZXRyaWMpCiAgcmV0dXJuKGNiaW5kKHRlc3RfZGF0YSwgZGVsdGFfbG9nX2xpaz1kZWx0YV9sb2dfbGlrKSkKfQojIyMjIwojIERlZmluZSByZWdyZXNzaW9uIGZvcm11bGFlLgoKIyBSZWdyZXNzaW9uIGNvZGUgdG8gZml0IEdBTSBtb2RlbHMuCiNiYXNlbGluZV9ydF9yZWdyZXNzaW9uID0gcHN5Y2hvbWV0cmljIH4gdGUoZnJlcSwgbGVuLCBicyA9ICJjciIpICsgdGUocHJldl9mcmVxLCBwcmV2X2xlbiwgYnMgPSAiY3IiKSArIHRlKHByZXYyX2ZyZXEsIHByZXYyX2xlbiwgYnMgPSAiY3IiKQojYmFzZWxpZV9zcHJ0X3JlZ3Jlc3Npb24gPSBwc3ljaG9tZXRyaWMgfiB0ZShmcmVxLCBsZW4sIGJzID0gImNyIikgKyB0ZShwcmV2X2ZyZXEsIHByZXZfbGVuLCBicyA9ICJjciIpICsgdGUocHJldjJfZnJlcSwgcHJldjJfbGVuLCBicyA9ICJjciIpICsgdGUocHJldjNfZnJlcSwgcHJldjNfbGVuLCBicyA9ICJjciIpICsgdGUocHJldjRfZnJlcSwgcHJldjRfbGVuLCBicyA9ICJjciIpCgojZnVsbF9ydF9yZWdyZXNzaW9uID0gcHN5Y2hvbWV0cmljIH4gcyhzdXJwcmlzYWwsIGJzID0gImNyIiwgayA9IDIwKSArIHMocHJldl9zdXJwLCBicyA9ICJjciIsIGsgPSAyMCkgKyBzKHByZXYyX3N1cnAsIGJzID0gImNyIiwgayA9IDIwKSArIHRlKGZyZXEsIGxlbiwgYnMgPSAiY3IiKSArIHRlKHByZXZfZnJlcSwgcHJldl9sZW4sIGJzID0gImNyIikgKyB0ZShwcmV2Ml9mcmVxLCBwcmV2Ml9sZW4sIGJzID0gImNyIikKI2Z1bGxfc3BydF9yZWdyZXNzaW9uID0gcHN5Y2hvbWV0cmljIH4gcyhzdXJwcmlzYWwsIGJzID0gImNyIiwgayA9IDIwKSArIHMocHJldl9zdXJwLCBicyA9ICJjciIsIGsgPSAyMCkgKyBzKHByZXYyX3N1cnAsIGJzID0gImNyIiwgayA9IDIwKSArIHMocHJldjNfc3VycCwgYnMgPSAiY3IiLCBrID0gMjApICsgcyhwcmV2NF9zdXJwLCBicyA9ICJjciIsIGsgPSAyMCkgKyB0ZShmcmVxLCBsZW4sIGJzID0gImNyIikgKyB0ZShwcmV2X2ZyZXEsIHByZXZfbGVuLCBicyA9ICJjciIpICsgdGUocHJldjJfZnJlcSwgcHJldjJfbGVuLCBicyA9ICJjciIpICsgdGUocHJldjNfZnJlcSwgcHJldjNfbGVuLCBicyA9ICJjciIpICsgdGUocHJldjRfZnJlcSwgcHJldjRfbGVuLCBicyA9ICJjciIpCgojIFJlZ3Jlc3Npb24gQ29kZSB0byBmaXQgbGluZWFyIG1vZGVscwpiYXNlbGluZV9ydF9yZWdyZXNzaW9uID0gcHN5Y2hvbWV0cmljIH4gZnJlcSArIHByZXZfZnJlcSArIHByZXYyX2ZyZXEgKyBsZW4gKyBwcmV2X2xlbiArIHByZXYyX2xlbgpiYXNlbGluZV9zcHJ0X3JlZ3Jlc3Npb24gPSBwc3ljaG9tZXRyaWMgfiBmcmVxICsgcHJldl9mcmVxICsgcHJldjJfZnJlcSArIHByZXYzX2ZyZXEgKyBwcmV2NF9mcmVxICsgbGVuICsgcHJldl9sZW4gKyBwcmV2Ml9sZW4gKyBwcmV2M19sZW4gKyBwcmV2NF9sZW4KCmZ1bGxfc3BydF9yZWdyZXNzaW9uID0gcHN5Y2hvbWV0cmljIH4gc3VycHJpc2FsICsgcHJldl9zdXJwICsgcHJldjJfc3VycCArIHByZXYzX3N1cnAgKyBwcmV2NF9zdXJwICsgZnJlcSArIHByZXZfZnJlcSArIHByZXYyX2ZyZXEgKyBwcmV2M19mcmVxICsgcHJldjRfZnJlcSArIGxlbiArIHByZXZfbGVuICsgcHJldjJfbGVuICsgcHJldjNfbGVuICsgcHJldjRfbGVuCmZ1bGxfcnRfcmVncmVzc2lvbiA9IHBzeWNob21ldHJpYyB+IHN1cnByaXNhbCArIHByZXZfc3VycCArIHByZXYyX3N1cnAgKyBmcmVxICsgcHJldl9mcmVxICsgcHJldjJfZnJlcSArIGxlbiArIHByZXZfbGVuICsgcHJldjJfbGVuCiAgCiMjIyMjCiMgUHJlcGFyZSBmcmFtZXMvZW52aXJvbm1lbnRzIGZvciBzdG9yaW5nIHJlc3VsdHMvb2JqZWN0cy4KYmFzZWxpbmVfcmVzdWx0cyA9IGRhdGEuZnJhbWUoKQpmdWxsX21vZGVsX3Jlc3VsdHMgPSBkYXRhLmZyYW1lKCkKYmFzZWxpbmVfcmVzaWR1YWxzID0gZGF0YS5mcmFtZSgpCmZ1bGxfcmVzaWR1YWxzID0gZGF0YS5mcmFtZSgpCmxvZ19saWtfZGVsdGFzID0gZGF0YS5mcmFtZSgpCgojUmFuZG9tbHkgc2h1ZmZsZSB0aGUgZGF0YQphbGxfZGF0YTwtYWxsX2RhdGFbc2FtcGxlKG5yb3coYWxsX2RhdGEpKSxdCiNDcmVhdGUgSyBlcXVhbGx5IHNpemUgZm9sZHMKSyA9IDEwCmZvbGRzIDwtIGN1dChzZXEoMSxucm93KGFsbF9kYXRhKSksYnJlYWtzPUssbGFiZWxzPUZBTFNFKQojUGVyZm9ybSAxMCBmb2xkIGNyb3NzIHZhbGlkYXRpb24KCiMgRml0IG1vZGVscyBmb3Igc29tZSBmb2xkIG9mIHRoZSBkYXRhLgpiYXNlbGluZV9jb3JwdXMgPSBmdW5jdGlvbihjb3JwdXMsIGRmLCB0ZXN0X2RhdGEsIGZvbGQsIGVudikgewogIGlmKGNvcnB1cyA9PSAiZHVuZGVlIikgewogICAgZ2V0X2xtX2RhdGEoZGYsIHRlc3RfZGF0YSwgYmFzZWxpbmVfcnRfcmVncmVzc2lvbiwgZm9sZCwgZW52KQogIH0gZWxzZSB7CiAgICBnZXRfbG1fZGF0YShkZiwgdGVzdF9kYXRhLCBiYXNlbGluZV9zcHJ0X3JlZ3Jlc3Npb24sIGZvbGQsIGVudikKICB9Cn0KZnVsbF9tb2RlbF9jb3JwdXMgPSBmdW5jdGlvbihjb3JwdXMsIGRmLCB0ZXN0X2RhdGEsIGZvbGQsIGVudikgewogIGlmKGNvcnB1c1sxXSA9PSAiZHVuZGVlIikgewogICAgZ2V0X2xtX2RhdGEoZGYsIHRlc3RfZGF0YSwgZnVsbF9ydF9yZWdyZXNzaW9uLCBmb2xkLCBlbnYpCiAgfSBlbHNlIHsKICAgIGdldF9sbV9kYXRhKGRmLCB0ZXN0X2RhdGEsIGZ1bGxfc3BydF9yZWdyZXNzaW9uLCBmb2xkLCBlbnYpCiAgfQp9CgojIFByZXBhcmUgYSBuZXcgRW52aXJvbm1lbnQgaW4gd2hpY2ggd2Ugc3RvcmUgZml0dGVkIExNcywgd2hpY2ggd2UnbGwgcXVlcnkgbGF0ZXIgZm9yIHJlc2lkdWFscyBhbmQgb3RoZXIgbWV0cmljcy4KYmFzZWxpbmVfZW52ID0gbmV3LmVudigpCmZ1bGxfZW52ID0gbmV3LmVudigpCgpmb3IoaSBpbiAxOkspIHsgCiAgI1NlZ2VtZW50IHlvdXIgZGF0YSBieSBmb2xkIHVzaW5nIHRoZSB3aGljaCgpIGZ1bmN0aW9uIAogIHRlc3RJbmRleGVzIDwtIHdoaWNoKGZvbGRzPT1pLCBhcnIuaW5kPVRSVUUpCiAgdGVzdF9kYXRhIDwtIGFsbF9kYXRhW3Rlc3RJbmRleGVzLCBdCiAgdHJhaW5fZGF0YSA8LSBhbGxfZGF0YVstdGVzdEluZGV4ZXMsIF0KICAKICAjIENvbXB1dGUgYSBiYXNlbGluZSBsaW5lYXIgbW9kZWwgZm9yIGVhY2ggbW9kZWwtLXRyYWluaW5nLS1zZWVkLS1SVC1jb3JwdXMgY29tYmluYXRpb24uCiAgYmFzZWxpbmVzID0gdHJhaW5fZGF0YSAlPiUKICAgIGdyb3VwX2J5KG1vZGVsLCB0cmFpbmluZywgc2VlZCwgY29ycHVzKSAlPiUKICAgICAgcHJpbnQobW9kZWwpICU+JQogICAgICBkbyhiYXNlbGluZV9jb3JwdXModW5pcXVlKC4kY29ycHVzKSwgLiwgdGVzdF9kYXRhLCBpLCBiYXNlbGluZV9lbnYpKSAlPiUKICAgIHVuZ3JvdXAoKSAlPiUKICAgIG11dGF0ZShzZWVkID0gYXMuZmFjdG9yKHNlZWQpLAogICAgICAgICAgIGZvbGQgPSBpKQogIAogIGJhc2VsaW5lX3Jlc3VsdHMgPSByYmluZChiYXNlbGluZV9yZXN1bHRzLCBiYXNlbGluZXMpCiAgCiAgIyBDb21wdXRlIGEgZnVsbCBsaW5lYXIgbW9kZWwgZm9yIGVhY2ggbW9kZWwtLXRyYWluaW5nLS1zZWVkLVJULWNvcnB1cyBjb21iaW5hdGlvbgogIGZ1bGxfbW9kZWxzID0gdHJhaW5fZGF0YSAlPiUKICAgIGdyb3VwX2J5KG1vZGVsLCB0cmFpbmluZywgc2VlZCwgY29ycHVzKSAlPiUKICAgICAgZG8oZnVsbF9tb2RlbF9jb3JwdXModW5pcXVlKC4kY29ycHVzKSwgLiwgdGVzdF9kYXRhLCBpLCBmdWxsX2VudikpICU+JQogICAgdW5ncm91cCgpICU+JQogICAgbXV0YXRlKHNlZWQgPSBhcy5mYWN0b3Ioc2VlZCksCiAgICAgICAgICAgZm9sZCA9IGkpCiAgCiAgZnVsbF9tb2RlbF9yZXN1bHRzID0gcmJpbmQoZnVsbF9tb2RlbF9yZXN1bHRzLCBmdWxsX21vZGVscykKICAKICAjIENvbXB1dGUgZGVsdGEtbG9nLWxpa2VsaWhvb2RzCiAgZm9sZF9sb2dfbGlrX2RlbHRhcyA9IHRlc3RfZGF0YSAlPiUKICAgIGdyb3VwX2J5KG1vZGVsLCB0cmFpbmluZywgc2VlZCwgY29ycHVzKSAlPiUKICAgICAgZG8oZ2V0X2xtX2RlbHRhX2xvZ19saWsoLiwgaSwgYmFzZWxpbmVfZW52LCBmdWxsX2VudikpICU+JQogICAgdW5ncm91cCgpCgogIGxvZ19saWtfZGVsdGFzID0gcmJpbmQobG9nX2xpa19kZWx0YXMsIGZvbGRfbG9nX2xpa19kZWx0YXMpCiAgCiAgZm9sZF9iYXNlbGluZV9yZXNpZHVhbHMgPSB0ZXN0X2RhdGEgJT4lCiAgICBncm91cF9ieShtb2RlbCwgdHJhaW5pbmcsIHNlZWQsIGNvcnB1cykgJT4lCiAgICAgIGRvKGdldF9sbV9yZXNpZHVhbHMoLiwgaSwgYmFzZWxpbmVfZW52KSkgJT4lCiAgICB1bmdyb3VwKCkKCiAgYmFzZWxpbmVfcmVzaWR1YWxzID0gcmJpbmQoYmFzZWxpbmVfcmVzaWR1YWxzLCBmb2xkX2Jhc2VsaW5lX3Jlc2lkdWFscykKCiAgZm9sZF9mdWxsX3Jlc2lkdWFscyA9IHRlc3RfZGF0YSAlPiUKICAgIGdyb3VwX2J5KG1vZGVsLCB0cmFpbmluZywgc2VlZCwgY29ycHVzKSAlPiUKICAgICAgZG8oZ2V0X2xtX3Jlc2lkdWFscyguLCBpLCBmdWxsX2VudikpICU+JQogICAgdW5ncm91cCgpCgogIGZ1bGxfcmVzaWR1YWxzID0gcmJpbmQoZnVsbF9yZXNpZHVhbHMsIGZvbGRfZnVsbF9yZXNpZHVhbHMpCn0KYGBgCgpgYGB7cn0KI3dyaXRlLmNzdihmdWxsX3Jlc2lkdWFscywgIi4uL2RhdGEvYW5hbHlzaXNfY2hlY2twb2ludHMvZnVsbF9yZXNpZHVhbHMuY3N2IikKI3dyaXRlLmNzdihiYXNlbGluZV9yZXNpZHVhbHMsICIuLi9kYXRhL2FuYWx5c2lzX2NoZWNrcG9pbnRzL2Jhc2VsaW5lX3Jlc2lkdWFscy5jc3YiKQpgYGAKCmBgYHtyfQptb2RlbF9kZWx0YXMgPSBsb2dfbGlrX2RlbHRhcyAlPiUKICBncm91cF9ieShtb2RlbCwgdHJhaW5pbmcsIHNlZWQsIGNvcnB1cykgJT4lIAogIHN1bW1hcmlzZShtZWFuX2RlbHRhX2xvZ19saWsgPSBtZWFuKGRlbHRhX2xvZ19saWspLAogICAgICAgICAgICBzZW1fZGVsdGFfbG9nX2xpayA9IHNkKGRlbHRhX2xvZ19saWspIC8gc3FydChsZW5ndGgoZGVsdGFfbG9nX2xpaykpKQpgYGAKCmBgYHtyfQp3cml0ZS5jc3YoZnVsbF9tb2RlbF9yZXN1bHRzLCAiLi4vZGF0YS9hbmFseXNpc19jaGVja3BvaW50cy9mdWxsX21vZGVsX3Jlc3VsdC5jc3YiKQp3cml0ZS5jc3YoYmFzZWxpbmVfcmVzdWx0cywgIi4uL2RhdGEvYW5hbHlzaXNfY2hlY2twb2ludHMvYmFzZWxpbmVfcmVzdWx0cy5jc3YiKQojZnVsbF9tb2RlbF9yZXN1bHRzID0gcmVhZC5jc3YoIi4uL2RhdGEvYW5hbHlzaXNfY2hlY2twb2ludHMvZmZ1bGxfbW9kZWxfcmVzdWx0cy5jc3YiKQojYmFzZWxpbmVfcmVzdWx0cyA9IHJlYWQuY3N2KCIuLi9kYXRhL2FuYWx5c2lzX2NoZWNrcG9pbnRzL2ZiYXNlbGluZV9yZXN1bHRzYi5jc3YiKQpgYGAKCmBgYHtyfQptZXRyaWMgPC0gIs6UTG9nTGlrIgojbWV0cmljIDwtICItzpRNU0UiCgojICMgU2VsZWN0IHRoZSByZWxldmFudCBtZXRyaWMuCm1vZGVsX2RlbHRhcyA9IG1vZGVsX2RlbHRhcyAlPiUKICAgICMgUmV0cmlldmUgdGhlIGN1cnJlbnQgdGVzdCBtZXRyaWMKICAgIG11dGF0ZShkZWx0YV90ZXN0X21lYW4gPSBtZWFuX2RlbHRhX2xvZ19saWssCiAgICAgICAgICAgZGVsdGFfdGVzdF9zZW0gPSBzZW1fZGVsdGFfbG9nX2xpaykgJT4lCiAgICAjIG11dGF0ZShkZWx0YV90ZXN0X21lYW4gPSBtZWFuX2RlbHRhX21zZSwKICAgICMgICAgICAgIGRlbHRhX3Rlc3Rfc2VtID0gc2VtX2RlbHRhX21zZSkKICAgIAogICAgIyBSZW1vdmUgdGhlIHJhdyBtZXRyaWNzLgogICAgc2VsZWN0KC1tZWFuX2RlbHRhX2xvZ19saWssIC1zZW1fZGVsdGFfbG9nX2xpaywKICAgICAgICAgICAjLW1lYW5fZGVsdGFfbXNlLCAtc2VtX2RlbHRhX21zZQogICAgICAgICAgICkKbW9kZWxfZGVsdGFzCmBgYAoKYGBge3IsIGV2YWw9RmFsc2V9CiMgU2FuaXR5IGNoZWNrOiB0cmFpbmluZyBvbiB0cmFpbit0ZXN0IGRhdGEgc2hvdWxkIHlpZWxkIGltcHJvdmVkIHBlcmZvcm1hbmNlIG92ZXIgdHJhaW5pbmcgb24ganVzdCB0cmFpbmluZyBkYXRhLiAoV2hlbiBldmFsdWF0aW5nIG9uIHRlc3QgZGF0YS4pCiBmdWxsX2Jhc2VsaW5lcyA9IGFsbF9kYXRhICU+JQogICBncm91cF9ieShtb2RlbCwgdHJhaW5pbmcsIHNlZWQsIGNvcnB1cykgJT4lCiAgIHN1bW1hcmlzZShiYXNlbGluZV90cmFpbl9hbGxfdGVzdF9saWsgPSBsb2dMaWtfdGVzdChsbShwc3ljaG9tZXRyaWMgfiBsZW4gKyBmcmVxICsgc2VudF9wb3MsIGRhdGE9LiksIHNlbWlfam9pbih0ZXN0X2RhdGEsIC4sIGJ5PWMoInRyYWluaW5nIiwgIm1vZGVsIiwgInNlZWQiLCAiY29ycHVzIikpLCBzZW1pX2pvaW4odGVzdF9kYXRhLCAuLCBieT1jKCJ0cmFpbmluZyIsICJtb2RlbCIsICJzZWVkIiwgImNvcnB1cyIpKSRwc3ljaG9tZXRyaWMpKSAlPiUKICAgdW5ncm91cCgpCiBmdWxsX2Jhc2VsaW5lcwogCiBmdWxsX2Jhc2VsaW5lcyAlPiUKICAgcmlnaHRfam9pbihiYXNlbGluZXMsIGJ5PWMoInNlZWQiLCAidHJhaW5pbmciLCAibW9kZWwiLCAiY29ycHVzIikpICU+JQogICBtdXRhdGUoZGVsdGE9YmFzZWxpbmVfdHJhaW5fYWxsX3Rlc3RfbGlrLWJhc2VsaW5lX3Rlc3RfbGlrKSAlPiUKICAgc2VsZWN0KC1iYXNlbGluZV9saWspICMgJT4lCiAgICNzZWxlY3QoLWJhc2VsaW5lX3Rlc3RfbGlrLCAtYmFzZWxpbmVfdHJhaW5fYWxsX3Rlc3RfbGlrLCAtYmFzZWxpbmVfbGlrLCAtYmFzZWxpbmVfdGVzdF9tc2UpCmBgYAoKIyBMb2FkIGxhbmd1YWdlIG1vZGVsIGRhdGEgKFN5bnRheEd5bSwgUFBMKQoKYGBge3J9Cmxhbmd1YWdlX21vZGVsX2RhdGEgPSByZWFkLmNzdigiLi4vZGF0YS9tb2RlbF9tZXRhZGF0YS5jc3YiKSAlPiUKICBtdXRhdGUobW9kZWwgPSBhcy5jaGFyYWN0ZXIobW9kZWwpLAogICAgICAgICBtb2RlbCA9IGlmX2Vsc2UobW9kZWwgPT0gImdwdC0yIiwgImdwdDIiLCBtb2RlbCksCiAgICAgICAgIG1vZGVsID0gYXMuZmFjdG9yKG1vZGVsKSkgJT4lCiAgbXV0YXRlKHRyYWluX3NpemUgPSBjYXNlX3doZW4oc3RyX3N0YXJ0cyh0cmFpbmluZywgImJsbGlwLWxnIikgfiA0MiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzdHJfc3RhcnRzKHRyYWluaW5nLCAiYmxsaXAtbWQiKSB+IDE1LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHN0cl9zdGFydHModHJhaW5pbmcsICJibGxpcC1zbSIpIH4gNSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzdHJfc3RhcnRzKHRyYWluaW5nLCAiYmxsaXAteHMiKSB+IDEpLAogICAgICAgICAKICAgICAgICAgIyBUcmFpbmluZyB2b2NhYnVsYXJ5IHVzdWFsbHkgY292YXJpZXMgd2l0aCB0aGUgdHJhaW5pbmcgY29ycHVzLgogICAgICAgICAjIEJ1dCBCUEUgbW9kZWxzIHNoYXJlIGEgdm9jYWJ1bGFyeSBhY3Jvc3MgdHJhaW5pbmcgY29ycG9yYS4KICAgICAgICAgdHJhaW5pbmdfdm9jYWI9YXMuZmFjdG9yKGlmZWxzZShzdHJfZGV0ZWN0KHRyYWluaW5nLCAiZ3B0YnBlIiksICJncHRicGUiLCBhcy5jaGFyYWN0ZXIodHJhaW5pbmcpKSksCiAgICAgICAgIHRyYWluaW5nX3NvdXJjZT1hcy5mYWN0b3Ioc3RyX3JlcGxhY2UoYXMuY2hhcmFjdGVyKHRyYWluaW5nKSwgIi1ncHRicGUiLCAiIikpCiAgICAgICAgICkgJT4lCiAgbXV0YXRlKHNlZWQgPSBhcy5mYWN0b3Ioc2VlZCkpICU+JQogIHNlbGVjdCgtcGlkLCAtdGVzdF9sb3NzKSAlPiUKICBkaXN0aW5jdChtb2RlbCwgdHJhaW5pbmcsIHNlZWQsIC5rZWVwX2FsbCA9IFRSVUUpCnRhYmxlKGxhbmd1YWdlX21vZGVsX2RhdGEkc2VlZCkKdGFibGUobW9kZWxfZGVsdGFzJHNlZWQpCmBgYAoKRmlyc3Qgam9pbiBkZWx0YS1tZXRyaWMgZGF0YSB3aXRoIG1vZGVsIGF1eGlsaWFyeSBkYXRhLgoKYGBge3J9Cm1vZGVsX2RlbHRhcyA9IG1vZGVsX2RlbHRhcyAlPiUKICBtZXJnZShsYW5ndWFnZV9tb2RlbF9kYXRhLCBieSA9IGMoInNlZWQiLCAidHJhaW5pbmciLCAibW9kZWwiKSwgYWxsPVQpICU+JQogIGRyb3BfbmEoKQoKbW9kZWxfZGVsdGFzCmBgYAoKQWxzbyBqb2luIG9uIHRoZSBvcmlnaW5hbCBsaW5lYXIgbW9kZWwgZGF0YSwgcmF0aGVyIHRoYW4gY29sbGFwc2luZyB0byBkZWx0YS1tZXRyaWNzLgpUaGlzIHdpbGwgc3VwcG9ydCByZWdyZXNzaW9ucyBsYXRlciBvbiB0aGF0IGRvbid0IGNvbGxhcHNlIGFjcm9zcyBmb2xkcy4KCgojIEZpbmFsIGRhdGEgcHJlcHJvY2Vzc2luZwoKYGBge3IgRmlsdGVyIG1vZGVscyBhbmQvb3IgY29ycG9yYX0KIyBFeGNsdWRlIG9yZGVyZWQtbmV1cm9ucyBmcm9tIGFsbCBhbmFseXNlcy4KbW9kZWxfZGVsdGFzIDwtIG1vZGVsX2RlbHRhcyAlPiUKICBmaWx0ZXIobW9kZWwgIT0gIm9yZGVyZWQtbmV1cm9ucyIpCgojIEV4Y2x1ZGUgYmFkIEdQVCBtb2RlbHMuCm1vZGVsX2RlbHRhcyA8LSBtb2RlbF9kZWx0YXMgJT4lCiAgZmlsdGVyKG1vZGVsICE9ICJncHQyIiB8ICEoc2VlZCAlaW4lIGMoMTU4MTk1NTI4OCwgMTU4MTg2MTQ3NCwgMTU4MjEyNjMyMCkpKQpgYGAKCgojIFZpc3VhbGl6YXRpb25zCgojIyBUaGUgYmFzaWNzCgpgYGB7ciwgZmlnLmNhcD0iQ29ycHVzIHNpemVzIn0KYWxsX2RhdGEgJT4lIGdncGxvdChhZXMoeD1jb3JwdXMpKSArIGdlb21fYmFyKCkKcHJpbnQoYWxsX2RhdGEgJT4lIGdyb3VwX2J5KGNvcnB1cykgJT4lIHN1bW1hcmlzZShuPW4oKSkpCmBgYAoKCmBgYHtyLCBmaWcuY2FwPSJXb3JkIGZyZXF1ZW5jeSBkaXN0cmlidXRpb24gYnkgY29ycHVzIn0KYWxsX2RhdGEgJT4lIAogIGdncGxvdChhZXMoeD1mcmVxLCBjb2xvcj1jb3JwdXMpKSArIGdlb21fZGVuc2l0eSgpCmBgYAoKYGBge3IsIGZpZy5jYXA9IldvcmQgbGVuZ3RoIGRpc3RyaWJ1dGlvbiBieSBjb3JwdXMifQphbGxfZGF0YSAlPiUgCiAgZ2dwbG90KGFlcyh4PWxlbiwgY29sb3I9Y29ycHVzKSkgKyBnZW9tX2RlbnNpdHkoKQpgYGAKCmBgYHtyLCBmaWcuY2FwPSJTdXJwcmlzYWwgZGlzdHJpYnV0aW9uIGJ5IGNvcnB1cyJ9CmFsbF9kYXRhICU+JSAKICBnZ3Bsb3QoYWVzKHg9c3VycHJpc2FsLCBjb2xvcj1jb3JwdXMpKSArIGdlb21fZGVuc2l0eSgpCmBgYAoKIyMgUHJlZGljdGl2ZSBwb3dlciBhbmQgU0cKCgpgYGB7ciBCeSBtb2RlbH0KbW9kZWxfZGVsdGFzICU+JQogIGdncGxvdChhZXMoeD1zZ19zY29yZSwgeT1kZWx0YV90ZXN0X21lYW4pKSArCiAgICBnZW9tX2Vycm9yYmFyKGFlcyh5bWluPWRlbHRhX3Rlc3RfbWVhbi1kZWx0YV90ZXN0X3NlbSwgeW1heD1kZWx0YV90ZXN0X21lYW4rZGVsdGFfdGVzdF9zZW0pKSArCiAgICBnZW9tX3Ntb290aChtZXRob2Q9ImxtIiwgc2U9VCkgKwogICAgZ2VvbV9wb2ludChzdGF0PSJpZGVudGl0eSIsIHBvc2l0aW9uPSJkb2RnZSIsIGFscGhhPTEsIHNpemU9MywgYWVzKGNvbG9yPXRyYWluaW5nX3ZvY2FiLCBzaGFwZT1tb2RlbCkpICsKICAgIHlsYWIobWV0cmljKSArCiAgICB4bGFiKCJTeW50YXggR2VuZXJhbGl6YXRpb24gU2NvcmUiKSArCiAgICBnZ3RpdGxlKCJTeW50YWN0aWMgR2VuZXJhbGl6YXRpb24gdnMuIFByZWRpY3RpdmUgUG93ZXIiKSArCiAgICBzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzID0gYygiYmxsaXAtbGciPSIjNDQwMTU0RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtbWQiPSIjMzk1NjhDRkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtc20iPSIjMUY5NjhCRkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAteHMiPSIjNzNEMDU1RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZ3B0YnBlIj0iIzg4ODg4OCIpKSArCiAgICBmYWNldF9ncmlkKH5jb3JwdXMsIHNjYWxlcz0iZnJlZSIpICsKICAgIHRoZW1lKGF4aXMudGV4dD1lbGVtZW50X3RleHQoc2l6ZT0xNCksCiAgICAgICAgICBzdHJpcC50ZXh0LnggPSBlbGVtZW50X3RleHQoc2l6ZT0xNCksCiAgICAgICAgICBsZWdlbmQudGV4dD1lbGVtZW50X3RleHQoc2l6ZT0xNCksCiAgICAgICAgICBheGlzLnRpdGxlPWVsZW1lbnRfdGV4dChzaXplPTE4KSwKICAgICAgICAgIGxlZ2VuZC5wb3NpdGlvbiA9ICJib3R0b20iKQojZ2dzYXZlKCIuL2NvZ3NjaV9pbWFnZXMvc2dfbG9nbGlrLnBuZyIsaGVpZ2h0PTUsd2lkdGg9NikKYGBgCgojIyMgUmVncmVzc2lvbiBhbmFseXNlcwoKV2UgY29udHJvbCBmb3IgZWZmZWN0cyBvZiBwZXJwbGV4aXR5IGJ5IHJlbGF0aW5nIHRoZSByZXNpZHVhbHMgb2YgYSBgcGVyZm9ybWFuY2UgfiBQUExgIHJlZ3Jlc3Npb24gdG8gU0cgc2NvcmUuCgpgYGB7ciBSZXNpZHVhbGl6ZWQgcmVncmVzc2lvbn0KIyBQcmVwYXJlIGEgcmVzaWR1YWxpemVkIHJlZ3Jlc3Npb24gZm9yIHgxIG9udG8geSwgY29udHJvbGxpbmcgZm9yIHRoZSBlZmZlY3RzIG9mIHgyLgpkX3Jlc2lkID0gbW9kZWxfZGVsdGFzICU+JQogIGRyb3BfbmEoKSAlPiUKICAKICBncm91cF9ieShjb3JwdXMpICU+JQogICAgIyBSZXNpZHVhbGl6ZSBkZWx0YSBtZXRyaWMgdy5yLnQgUFBMIGZvciBlYWNoIG1vZGVsLS10cmFpbmluZy0tc2VlZCB3aXRoaW4KICAgICMgdHJhaW5pbmcgdm9jYWJ1bGFyeQogICAgbXV0YXRlKHJlc2lkLmRlbHRhID0gcmVzaWQobG0oZGVsdGFfdGVzdF9tZWFuIH4gdHJhaW5pbmdfdm9jYWI6dGVzdF9wcGwpKSkgJT4lCiAgICAjIFJlc2lkdWFsaXplIFNHIHNjb3JlIHcuci50LiBQUEwgZm9yIGVhY2ggbW9kZWwtLXRyYWluaW5nLS1zZWVkCiAgICAjIHdpdGhpbiB0cmFpbmluZyB2b2NhYnVsYXJ5CiAgICBtdXRhdGUocmVzaWQuc2cgPSByZXNpZChsbShzZ19zY29yZSB+IHRyYWluaW5nX3ZvY2FiOnRlc3RfcHBsKSkpICU+JQogIHVuZ3JvdXAoKQoKCiMgTm93IHBsb3QgcmVzaWR1YWwgdnMgU0cKZF9yZXNpZCAlPiUKICBnZ3Bsb3QoYWVzKHg9cmVzaWQuc2csIHk9cmVzaWQuZGVsdGEpKSArCiAgICB0aGVtZV9idygpICsKICAgIHNjYWxlX3NoYXBlX21hbnVhbCh2YWx1ZXMgPSBjKDIxLCAyNCwgMjIsIDIzKSkgKwogICAgZ2VvbV9zbW9vdGgobWV0aG9kPSJsbSIsIHNlPVQsIGFscGhhPTAuMykgKwogICAgZ2VvbV9wb2ludChzdGF0PSJpZGVudGl0eSIsIHBvc2l0aW9uPSJkb2RnZSIsIGFscGhhPTEsIHNpemU9NSwgYWVzKHNoYXBlID0gbW9kZWwsIGZpbGw9dHJhaW5pbmdfc291cmNlLCBjb2xvciA9IHRyYWluaW5nX3ZvY2FiLCBzdHJva2UgPSAxKSkgKwogICAgeWxhYihwYXN0ZSgiUmVzaWR1YWwiLCBtZXRyaWMpKSArCiAgICB4bGFiKCJSZXNpZHVhbCBTeW50YXggR2VuZXJhbGl6YXRpb24gU2NvcmUiKSArCiAgICBnZ3RpdGxlKCJTeW50YWN0aWMgR2VuZXJhbGl6YXRpb24gdnMuIFByZWRpY3RpdmUgUG93ZXIiKSArCiAgICBsYWJzKGNvbG9yPSJ0cmFpbmluZyIpICsgCiAgICBzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzID0gYygiYmxsaXAtbGciPSIjNDQwMTU0RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLW1kIj0iIzM5NTY4Q0ZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1zbSI9IiMxRjk2OEJGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAteHMiPSIjNzNEMDU1RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImdwdGJwZSI9IiNmMDk0MWYiKSkgKwogIHNjYWxlX2ZpbGxfbWFudWFsKHZhbHVlcyA9IGMoImJsbGlwLWxnIj0iIzQ0MDE1NEZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1tZCI9IiMzOTU2OENGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtc20iPSIjMUY5NjhCRkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLXhzIj0iIzczRDA1NUZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJncHRicGUiPSIjZjA5NDFmIiksIGd1aWRlPUYpICsKICAgIGZhY2V0X2dyaWQoLn5jb3JwdXMsIHNjYWxlcz0iZnJlZSIpICsKICAgIHRoZW1lKGF4aXMudGV4dD1lbGVtZW50X3RleHQoc2l6ZT0xNCksCiAgICAgICAgICBzdHJpcC50ZXh0LnggPSBlbGVtZW50X3RleHQoc2l6ZT0xNCksCiAgICAgICAgICBsZWdlbmQudGV4dD1lbGVtZW50X3RleHQoc2l6ZT0xNCksCiAgICAgICAgICBheGlzLnRpdGxlPWVsZW1lbnRfdGV4dChzaXplPTE4KSwKICAgICAgICAgIGxlZ2VuZC5wb3NpdGlvbiA9ICJyaWdodCIpCmdnc2F2ZSgiLi4vaW1hZ2VzL2NvZ3NjaTIwMjAvZGxsX3NnLnBkZiIsaGVpZ2h0PTQuNSx3aWR0aD05LCBkZXZpY2UgPSBjYWlyb19wZGYpCmBgYAoKCmBgYHtyIFN0ZXB3aXNlIHJlZ3Jlc3Npb259CmRvX3N0ZXB3aXNlX3JlZ3Jlc3Npb24gPSBmdW5jdGlvbihjdXJfY29ycHVzKSB7CiAgcmVncmVzc2lvbl9kYXRhID0gbW9kZWxfZGVsdGFzICU+JQogICAgZmlsdGVyKGNvcnB1cyA9PSBjdXJfY29ycHVzKQogIAogIHByaW50KCItLS0tLS0tLS0tLS0tLS0tLS0tLS0tIikKICBwcmludChjdXJfY29ycHVzKQogIAogIGxtMSA9IGxtKGRlbHRhX3Rlc3RfbWVhbiB+IHRyYWluaW5nX3ZvY2FiOnRlc3RfcHBsLCBkYXRhID0gcmVncmVzc2lvbl9kYXRhKQogIGxtMiA9IGxtKGRlbHRhX3Rlc3RfbWVhbiB+IHRyYWluaW5nX3ZvY2FiOnRlc3RfcHBsICsgc2dfc2NvcmUsIGRhdGEgPSByZWdyZXNzaW9uX2RhdGEpCiAgcHJpbnQoYW5vdmEobG0xLCBsbTIpKQogIHN1bW1hcnkobG0yKQp9CmRvX3N0ZXB3aXNlX3JlZ3Jlc3Npb24oImJuYy1icm93biIpCmRvX3N0ZXB3aXNlX3JlZ3Jlc3Npb24oImR1bmRlZSIpCmRvX3N0ZXB3aXNlX3JlZ3Jlc3Npb24oIm5hdHVyYWwtc3RvcmllcyIpCmBgYAoKYGBge3IgU2FuaXR5IGNoZWNrOiBlcXVpdmFsZW5jZSBiZXR3ZWVuIGFuYWx5c2VzLCBldmFsPUZhbHNlfQojIFRoZSByZXNpZHVhbGl6ZWQgYW5hbHlzaXMgYW5kIHRoZSBzdGVwd2lzZSByZWdyZXNzaW9uIGFuYWx5c2lzCiMgc2hvdWxkIHlpZWxkIHRoZSBzYW1lIGNvZWZmaWNpZW50cyBmb3IgdGhlIFNHIHNjb3JlIHZhcmlhYmxlLgojCiMgQmVsb3csIHdlIGNvbXB1dGUgdGhlIHNsb3BlIGNvZWZmaWNpZW50IGZvciB0aGUgU0cgdGVybSBpbiB0aGUKIyByZXNpZHVhbGl6ZWQgYW5hbHlzZXMuCiMKIyBUaGVzZSBjb2VmZmljaWVudHMgc2hvdWxkIG1hdGNoIHRob3NlIGZvdW5kIGluIHRoZSBzdGVwd2lzZQojIHJlZ3Jlc3Npb24gZm9yIGBzZ19zY29yZWAgYWJvdmUuCmRfcmVzaWQgJT4lIGdyb3VwX2J5KGNvcnB1cykgJT4lCiAgZ3JvdXBfbW9kaWZ5KH50aWR5KGxtKHJlc2lkLmRlbHRhIH4gdHJhaW5pbmdfdm9jYWI6dGVzdF9wcGwgKyByZXNpZC5zZywgZGF0YT0uKSkKICAgICAgICAgICAgICAgICAlPiUgZmlsdGVyKHRlcm0gPT0gInJlc2lkLnNnIikpICU+JSAKICBzZWxlY3QoY29ycHVzLCBlc3RpbWF0ZSkKYGBgCgojIyBQcmVkaWN0aXZlIHBvd2VyIGFuZCBwZXJwbGV4aXR5CgpgYGB7cn0KCm1vZGVsX2RlbHRhcyAlPiUKICBtdXRhdGUodGVzdF9wcGwgPSBpZl9lbHNlKHRlc3RfcHBsID4gNTAwLCAzMjkuOSwgdGVzdF9wcGwpLAogICAgICAgICBicGUgPSBpZl9lbHNlKHRyYWluaW5nX3ZvY2FiID09ICJncHRicGUiLCAieWVzIiwgIm5vIikpICU+JQogIGdncGxvdChhZXMoeD10ZXN0X3BwbCwgeT1kZWx0YV90ZXN0X21lYW4sIHNoYXBlID0gbW9kZWwsIHltaW49MCkpICsKICAgIHRoZW1lX2J3KCkgKwogICAgZ2VvbV90ZXh0KGFlcyh4PTI3NSwgeT0wLCBsYWJlbCA9IGMoIi8vIikpKSArCiAgICBnZW9tX2Vycm9yYmFyKGFlcyh5bWluPWRlbHRhX3Rlc3RfbWVhbi1kZWx0YV90ZXN0X3NlbSwgeW1heD1kZWx0YV90ZXN0X21lYW4rZGVsdGFfdGVzdF9zZW0sIGNvbG9yPXRyYWluaW5nX3ZvY2FiKSwgYWxwaGE9MC40KSArCiAgICAjZ2VvbV9zbW9vdGgobWV0aG9kPSJsbSIsIHNlPUYpICsKICAgIGdlb21fcG9pbnQoc3RhdD0iaWRlbnRpdHkiLCBwb3NpdGlvbj0iZG9kZ2UiLCBhbHBoYT0xLCBzaXplPTUsIGFlcyhmaWxsPXRyYWluaW5nX3NvdXJjZSwgY29sb3IgPSB0cmFpbmluZ192b2NhYiwgc3Ryb2tlID0gMSkpICsKICAgIHlsYWIoIs6UTG9nTGlrIHBlciB0b2tlbiIpICsKICAgIHhsYWIoIlRlc3QgUGVycGxleGl0eSIpICsKICAgICNjb29yZF9jYXJ0ZXNpYW4oeWxpbSA9IGMoMSwgMTYpKSArCiAgICBnZ3RpdGxlKCJUZXN0IFBlcnBsZXhpdHkgdnMuIFByZWRpY3RpdmUgUG93ZXIiKSArCiAgICBsYWJzKGNvbG9yPSJ0cmFpbmluZyIpICsgCiAgICBzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzID0gYygiYmxsaXAtbGciPSIjNDQwMTU0RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLW1kIj0iIzM5NTY4Q0ZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1zbSI9IiMxRjk2OEJGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAteHMiPSIjNzNEMDU1RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImdwdGJwZSI9IiNmMDk0MWYiKSkgKwogIHNjYWxlX2ZpbGxfbWFudWFsKHZhbHVlcyA9IGMoImJsbGlwLWxnIj0iIzQ0MDE1NEZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1tZCI9IiMzOTU2OENGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtc20iPSIjMUY5NjhCRkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLXhzIj0iIzczRDA1NUZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJncHRicGUiPSIjZjA5NDFmIiksIGd1aWRlPUYpICsKICAgIHNjYWxlX3NoYXBlX21hbnVhbCh2YWx1ZXMgPSBjKDIxLCAyNCwgMjIsIDIzKSkgKwogICAgc2NhbGVfeF9jb250aW51b3VzKGxhYmVscz1jKDAsIDUwLCAxMDAsIDE1MCwgMjAwLCAyNTAsIDUwMCAsNTUwKSwgYnJlYWtzPWMoMCwgNTAsIDEwMCwgMTUwLCAyMDAsIDI1MCwgMzAwLCAzNTApLCBtaW5vcl9icmVha3MgPSBOVUxMKSArCiAgICBzY2FsZV95X2NvbnRpbnVvdXMobGltaXRzID0gYygwLCBOQSksIGV4cGFuZCA9IGMoMCwwKSkgKwogICAgZmFjZXRfd3JhcCh+Y29ycHVzLCBzY2FsZXM9ImZyZWUiKSArCiAgICBjb29yZF9jYXJ0ZXNpYW4oY2xpcD0ib2ZmIikgKwogICAgdGhlbWUoYXhpcy50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTEyKSwKICAgICAgICAgIHN0cmlwLnRleHQueCA9IGVsZW1lbnRfdGV4dChzaXplPTEyKSwKICAgICAgICAgIGxlZ2VuZC50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTEyKSwKICAgICAgICAgIGF4aXMudGl0bGU9ZWxlbWVudF90ZXh0KHNpemU9MTIpLAogICAgICAgICAgbGVnZW5kLnBvc2l0aW9uID0gInJpZ2h0IikKZ2dzYXZlKCIuLi9pbWFnZXMvY29nc2NpMjAyMC9wcGxfbG9nbGlrLnBkZiIsaGVpZ2h0PTUsd2lkdGg9MTIsIGRldmljZSA9IGNhaXJvX3BkZikKCmBgYAoKIyMjIFJlZ3Jlc3Npb246IEltcGFjdCBvZiBQUEwgb24gUHJlZGljdGl2ZSBQb3dlcgoKYGBge3J9CmxtZCA9IG1vZGVsX2RlbHRhcyAlPiUKICBtdXRhdGUodHJhaW5pbmdfdm9jYWI9aWZlbHNlKHN0cl9kZXRlY3QoYXMuY2hhcmFjdGVyKHRyYWluaW5nKSwgImdwdGJwZSIpLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImdwdGJwZSIsIGFzLmNoYXJhY3Rlcih0cmFpbmluZykpKQpzdW1tYXJ5KGxtZXIoZGVsdGFfdGVzdF9tZWFuIH4gdHJhaW5pbmdfdm9jYWI6dGVzdF9wcGwgKyAoMSB8IGNvcnB1cykgKyAoMSB8IG1vZGVsKSwgZGF0YT1sbWQpKQpgYGAKCiMjIFBlcnBsZXhpdHkgdnMuIFNHIFNjb3JlClRoaXMgaXMgYSByZXByb2R1Y3Rpb24gb2YgRmlndXJlIDIgZnJvbSBIdSBldCBhbC4KCmBgYHtyIFBQTCB2cy4gU0cgc2NvcmV9Cgptb2RlbF9kZWx0YXMgJT4lCiAgbXV0YXRlKHRlc3RfcHBsID0gaWZfZWxzZSh0ZXN0X3BwbCA+IDUwMCwgMzI5LjksIHRlc3RfcHBsKSkgJT4lCiAgbXV0YXRlKHRyYWluX3NpemUgPSBsb2codHJhaW5fc2l6ZSkpICU+JQogIG11dGF0ZShicGUgPSBpZl9lbHNlKHRyYWluaW5nX3ZvY2FiID09ICJncHRicGUiLCAieWVzIiwgIm5vIiksCiAgICAgICAgIGJwZSA9IGFzLmZhY3RvcihicGUpKSAlPiUKICBnZ3Bsb3QoYWVzKHg9dGVzdF9wcGwsIHk9c2dfc2NvcmUpKSArCiAgICB0aGVtZV9idygpICsKICAgIGdlb21faGxpbmUoeWludGVyY2VwdCA9IDAuMjgsIGxpbmV0eXBlID0gImRhc2hlZCIsIGNvbG9yPSJncmF5IikgKwogICAgZ2VvbV90ZXh0KGFlcyh4PTI0MCwgeT0wLjMpLCBsYWJlbD0icmFuZG9tIiwgY29sb3I9ImdyYXkiKSArCiAgICBnZW9tX3BvaW50KHN0YXQ9ImlkZW50aXR5IiwgcG9zaXRpb249ImRvZGdlIiwgYWxwaGE9MC4zLCBzaXplPTQsIGFlcyhzaGFwZSA9IG1vZGVsLCBmaWxsPXRyYWluaW5nX3NvdXJjZSwgY29sb3IgPSB0cmFpbmluZ192b2NhYiwgc3Ryb2tlID0gMSkpICsKICAgIGdlb21fdGV4dChhZXMoeD0yNzUsIHk9MCwgbGFiZWwgPSBjKCIvLyIpKSkgKwogICAgeWxhYigiU0cgU2NvcmUiKSArCiAgICB4bGFiKCJUZXN0IFBlcnBsZXhpdHkiKSArCiAgICBnZ3RpdGxlKCJUZXN0IFBQTCB2cy4gU0cgU2NvcmUiKSArCiAgICBsYWJzKGNvbG9yPSJ0cmFpbmluZyIpICsgCiAgICBzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzID0gYygiYmxsaXAtbGciPSIjNDQwMTU0RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLW1kIj0iIzM5NTY4Q0ZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1zbSI9IiMxRjk2OEJGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAteHMiPSIjNzNEMDU1RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImdwdGJwZSI9IiNmMDk0MWYiKSkgKwogICAgc2NhbGVfZmlsbF9tYW51YWwodmFsdWVzID0gYygiYmxsaXAtbGciPSIjNDQwMTU0RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLW1kIj0iIzM5NTY4Q0ZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1zbSI9IiMxRjk2OEJGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAteHMiPSIjNzNEMDU1RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImdwdGJwZSI9IiNmMDk0MWYiKSwgZ3VpZGU9RikgKwogICAgc2NhbGVfc2hhcGVfbWFudWFsKHZhbHVlcyA9IGMoIjVncmFtIj0yMSwgdmFuaWxsYT0yMiwgZ3B0Mj0yNCwgcm5uZz0yMykpICsKICAgIHNjYWxlX3hfY29udGludW91cyhsYWJlbHM9YygwLCA1MCwgMTAwLCAxNTAsIDIwMCwgMjUwLCA1MDAgLDU1MCksIGJyZWFrcz1jKDAsIDUwLCAxMDAsIDE1MCwgMjAwLCAyNTAsIDMwMCwgMzUwKSwgbWlub3JfYnJlYWtzID0gTlVMTCkgKwogICAgc2NhbGVfeV9jb250aW51b3VzKGxpbWl0cyA9IGMoMCwgMSksIGV4cGFuZCA9IGMoMCwwKSkgKwogICAgdGhlbWUoYXhpcy50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTEyKSwKICAgICAgICAgIHN0cmlwLnRleHQueCA9IGVsZW1lbnRfdGV4dChzaXplPTEyKSwKICAgICAgICAgIGxlZ2VuZC50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTgpLAogICAgICAgICAgbGVnZW5kLnRpdGxlPWVsZW1lbnRfdGV4dChzaXplPTgpLAogICAgICAgICAgYXhpcy50aXRsZT1lbGVtZW50X3RleHQoc2l6ZT0xNCksCiAgICAgICAgICBsZWdlbmQucG9zaXRpb24gPSAibm9uZSIsCiAgICAgICAgICBsZWdlbmQuZGlyZWN0aW9uID0gImhvcml6b250YWwiLAogICAgICAgICAgbGVnZW5kLmtleS53aWR0aCA9IHVuaXQoMC4zLCJjbSIpLAogICAgICAgICAgbGVnZW5kLnNwYWNpbmcueCA9IHVuaXQoMC4xLCAnY20nKSkKZ2dzYXZlKCIuLi9pbWFnZXMvY29nc2NpMjAyMC9wcGxfc2cucGRmIixoZWlnaHQ9NC41LHdpZHRoPTMsIGRldmljZSA9IGNhaXJvX3BkZikKYGBgCgoKIyMgU21pdGggJiBMZXZ5IHJlcHJvZHVjdGlvbgoKIyMjIyBUaGlzIHJlZG9uZSBzbyB0aGF0IGl0J3MgdW5pcXVlIGZvciBlYWNoIG1vZGVsCmBgYHtyLCBldmFsPUZhbHNlfQphbGxfZGF0YSAlPiUKICBnZ3Bsb3QoYWVzKHg9c3VycHJpc2FsLCBjb2xvcj1tb2RlbCkpICsKICB0aGVtZV9idygpICsKICBnZW9tX2RlbnNpdHkoKSArCiAgZmFjZXRfd3JhcCgufmNvcnB1cywgbmNvbD0xLCBzY2FsZXM9ImZyZWUiLCBzdHJpcC5wb3NpdGlvbiA9ICJyaWdodCIpICsKICBjb29yZF9jYXJ0ZXNpYW4oeGxpbSA9IGMoMCwgMjUpKSArCiAgZ2d0aXRsZSgiRGlzdHJpYnV0aW9uIG9mIFN1cnByaXNhbCIpICsKZ2dzYXZlKCIuLi9pbWFnZXMvY29nc2NpMjAyMC9zdXJwX2NvcnJfbWFyZ2luYWxzLnBuZyIsaGVpZ2h0PTUsd2lkdGg9NCkKCmBgYAoKYGBge3IgRml0IEdBTXMsIGV2YWw9RmFsc2V9CmsgPSAxLjk3CgojIEZpdCBhIEdBTSBmb3IgYSBib290c3RyYXAgc2FtcGxlLgpmaXRfZ2FtX2lubmVyID0gZnVuY3Rpb24oYm9vdHN0cmFwX3NhbXBsZSwga2V5KSB7CiAgIyBUaGlzIGJvb3RzdHJhcCBzYW1wbGUgbWF5IGhhdmUgcmVwZWF0ZWQgZWxlbWVudHMuIFRoYXQgY2F1c2VzIGEgcHJvYmxlbSBmb3IKICAjIG1nY3YsIHdoaWNoIGludGVybmFsbHkgY3Jvc3MtdmFsaWRhdGVzIHNvbWUgbW9kZWwgcGFyYW1ldGVycyAtLSBpdCBtYXkKICAjIGFsbG9jYXRlIHJlcGVhdGVkIGVsZW1lbnRzIHRvIGRpZmZlcmVudCBmb2xkcyBhbmQgdGh1cyBkb3VibGUtZGlwLiBXZSdsbAogICMgcHJldmVudCB0aGlzIGJ5IGluc3RlYWQgcHJvdmlkaW5nIHRoZSB3aG9sZSAocHJlLWJvb3RzdHJhcCkgZGF0YXNldCB0byBtZ2N2LAogICMgYW5kIHVzaW5nIGB3ZWlnaHRzYCB0byBjb25zdHJhaW4gd2hpY2ggZWxlbWVudHMgYXJlIHNlZW4sIGFuZCBob3cgbWFueQogICMgdGltZXMuIChSZXBlYXRlZCBlbGVtZW50cyBvZiB0aGUgc2FtcGxlIG1heSBnZXQgYSB3ZWlnaHQgb2YgMiBvciAzIG9yIE4sCiAgIyB3aGljaCBpcyBleGFjdGx5IHdoYXQgd2Ugd2FudC4pCiAgCiAgIyByc3BsaXQkZGF0YSBjb250YWlucyB0aGUgb3JpZ2luYWwgZW50aXJlIGRhdGFzZXQuCiAgZGYgPSBib290c3RyYXBfc2FtcGxlJGRhdGEKICAjIGFzLmludGVnZXIucnNwbGl0IHJldHVybnMgdGhlIGluZGljZXMgb2YgdGhlIGV4YW1wbGVzIHdoaWNoIGFyZSBpbi1zYW1wbGUuCiAgIyBjb252ZXJ0IHRoaXMgdG8gYSBjb3VudCB2ZWN0b3IsIHdpdGggZGltZW5zaW9uIE4gKHRvdGFsIGRhdGFzZXQgcm93cykKICB3ZWlnaHRzID0gdGFidWxhdGUoYXMuaW50ZWdlcihib290c3RyYXBfc2FtcGxlKSwgbnJvdyhkZikpCiAgCiAgaWYgKGtleSRjb3JwdXMgPT0gImR1bmRlZSIpIHsKICAgICMgUmVhZGluZyB0aW1lIHJlZ3Jlc3Npb246IHVzZSBmZWF0dXJlcyBvZiBjdXJyZW50IGFuZCBwcmV2aW91cyB3b3JkCiAgICBtID0gZ2FtKHBzeWNob21ldHJpYyB+IHMoc3VycHJpc2FsLCBicyA9ICdjcicsIGsgPSAyMCkgKyBzKHByZXZfc3VycCwgYnMgPSAnY3InLCBrID0gMjApICsKICAgICAgICAgICAgICAgICAgICAgICAgICAgdGUoZnJlcSwgbGVuLCBicyA9ICdjcicpICsgdGUocHJldl9mcmVxLCBwcmV2X2xlbiwgYnMgPSAnY3InKSwKICAgICAgICAgICAgZGF0YSA9IGRmLCB3ZWlnaHRzID0gd2VpZ2h0cykKICAgIAogICAgdGVybXNfdG9fcHJlZGljdCA9IGMoInMoc3VycHJpc2FsKSIsICJzKHByZXZfc3VycCkiKQogIH0gZWxzZSB7CiAgICAjIFNQUlQgcmVncmVzc2lvbjogdXNlIGZlYXR1cmVzIG9mIGN1cnJlbnQgYW5kIDMgcHJldmlvdXMgd29yZHMKICAgIG0gPSBnYW0ocHN5Y2hvbWV0cmljIH4gcyhzdXJwcmlzYWwsIGJzID0gJ2NyJywgayA9IDIwKSArIHMocHJldl9zdXJwLCBicyA9ICdjcicsIGsgPSAyMCkgKwogICAgICAgICAgICAgICAgICAgICAgICAgICBzKHByZXYyX3N1cnAsIGJzID0gJ2NyJywgayA9IDIwKSArIHMocHJldjNfc3VycCwgYnMgPSAnY3InLCBrID0gMjApICsKICAgICAgICAgICAgICAgICAgICAgICAgICAgdGUoZnJlcSwgbGVuLCBicyA9ICdjcicpICsgdGUocHJldl9mcmVxLCBwcmV2X2xlbiwgYnMgPSAnY3InKSArCiAgICAgICAgICAgICAgICAgICAgICAgICAgIHRlKHByZXYyX2ZyZXEsIHByZXYyX2xlbiwgYnMgPSAnY3InKSArIHRlKHByZXYzX2ZyZXEsIHByZXYzX2xlbiwgYnMgPSAnY3InKSwKICAgICAgICAgICAgZGF0YSA9IGRmLCB3ZWlnaHRzID0gd2VpZ2h0cykKICAgIAogICAgdGVybXNfdG9fcHJlZGljdCA9IGMoInMoc3VycHJpc2FsKSIsICJzKHByZXZfc3VycCkiLAogICAgICAgICAgICAgICAgICAgICAgICAgInMocHJldjJfc3VycCkiLCAicyhwcmV2M19zdXJwKSIpCiAgfQoKICAjIFByb2R1Y2UgcHN5Y2hvbWV0cmljIHByZWRpY3Rpb25zIGxpbmUgdXNpbmcganVzdCB0aGUgcmVsZXZhbnQgY29udGV4dC1zcGVjaWZpYyBwcmVkaWN0b3JzLgoKICBuZXdkYXRhID0gZGF0YS5mcmFtZShzdXJwcmlzYWw9c2VxKDAsMjAsYnk9MC4xKSwKICAgICAgICAgICAgICAgICAgICAgICBwcmV2X3N1cnA9c2VxKDAsMjAsYnk9MC4xKSwKICAgICAgICAgICAgICAgICAgICAgICBwcmV2Ml9zdXJwPXNlcSgwLDIwLGJ5PTAuMSksCiAgICAgICAgICAgICAgICAgICAgICAgcHJldjNfc3VycD1zZXEoMCwyMCxieT0wLjEpLAogICAgICAgICAgICAgICAgICAgICAgIGZyZXE9MCwgcHJldl9mcmVxPTAsIHByZXYyX2ZyZXE9MCwgcHJldjNfZnJlcT0wLAogICAgICAgICAgICAgICAgICAgICAgIGxlbj0wLCBwcmV2X2xlbj0wLCBwcmV2Ml9sZW49MCwgcHJldjNfbGVuPTApCiAgCiAgIyBSZXR1cm5zIGEgbWF0cml4IE5fc2FtcGxlcyAqIE5fdGVybXMuCiAgcGVyX3Rlcm1fcHJlZGljdGlvbnMgPSBwcmVkaWN0KG0sIG5ld2RhdGE9bmV3ZGF0YSwgdGVybXM9dGVybXNfdG9fcHJlZGljdCwgdHlwZT0idGVybXMiKQogIAogICMgQWRkaXRpdmUgbW9kZWwgLS0gc3VtIGFjcm9zcyBwcmVkaWN0b3IgcmVzcG9uc2UgY29udHJpYnV0aW9ucyAobWF0cml4IGNvbHVtbnMpLgogIHByZWRpY3Rpb25zID0gcm93U3VtcyhwZXJfdGVybV9wcmVkaWN0aW9ucykKICAKICByZXR1cm4obmV3ZGF0YSAlPiUgbXV0YXRlKHk9cHJlZGljdGlvbnMpKQp9CgojIEZpdCBhIGJvb3RzdHJhcC1yZS1lc3RpbWF0ZWQgR0FNIGZvciB0aGUgZ2l2ZW4gbW9kZWwtLWNvcnB1cy0tdHJhaW5pbmcgZ3JvdXAuCmZpdF9nYW0gPSBmdW5jdGlvbihkZiwga2V5LCBhbHBoYT0wLjA1KSB7CiAgIyBCb290c3RyYXAtcmVzYW1wbGUgZGF0YQogIGJvb3RfbW9kZWxzID0gZGYgJT4lIGJvb3RzdHJhcHModGltZXM9NTApICU+JSAKICAgICMgRml0IGEgR0FNIGFuZCBnZXQgcHJlZGljdGlvbnMgZm9yIGVhY2ggc2FtcGxlCiAgICBtdXRhdGUoc21vb3RoZWQ9bWFwKHNwbGl0cywgZml0X2dhbV9pbm5lciwga2V5PWtleSkpCiAgCiAgIyBFeHRyYWN0IG1lYW4gYW5kIDUlIGFuZCA5NSUgcGVyY2VudGlsZSB5LXZhbHVlcyBmb3IgZWFjaCBzdXJwcmlzYWwgdmFsdWUKICByZXN1bHQgPSBib290X21vZGVscyAlPiUgCiAgICB1bm5lc3Qoc21vb3RoZWQpICU+JSAKICAgIHNlbGVjdChzdXJwcmlzYWwsIHkpICU+JSAKICAgIGdyb3VwX2J5KHN1cnByaXNhbCkgJT4lIAogICAgICBzdW1tYXJpc2UoeV9sb3dlcj1xdWFudGlsZSh5LCBhbHBoYSAvIDIpLCAKICAgICAgICAgICAgICAgIHlfdXBwZXI9cXVhbnRpbGUoeSwgMSAtIGFscGhhIC8gMiksCiAgICAgICAgICAgICAgICB5PW1lYW4oeSkpICU+JSAKICAgIHVuZ3JvdXAoKQogIAogIHJldHVybiAocmVzdWx0KQp9CgpzbW9vdGhzID0gYWxsX2RhdGEgJT4lCiAgbXV0YXRlKAogICAgdHJhaW5pbmdfdm9jYWI9YXMuZmFjdG9yKGlmZWxzZShzdHJfZGV0ZWN0KHRyYWluaW5nLCAiZ3B0YnBlIiksICJncHRicGUiLCBhcy5jaGFyYWN0ZXIodHJhaW5pbmcpKSksCiAgICB0cmFpbmluZ19zb3VyY2U9YXMuZmFjdG9yKHN0cl9yZXBsYWNlKGFzLmNoYXJhY3Rlcih0cmFpbmluZyksICItZ3B0YnBlIiwgIiIpKSkgJT4lCiAgZ3JvdXBfYnkodHJhaW5pbmdfdm9jYWIsIHRyYWluaW5nX3NvdXJjZSwgbW9kZWwsIGNvcnB1cykgJT4lCiAgICBncm91cF9tb2RpZnkoZml0X2dhbSkgJT4lCiAgdW5ncm91cCgpCndyaXRlLmNzdihzbW9vdGhzLCAiLi4vZGF0YS9nYW1fc21vb3Rocy5jc3YiKQoKCmBgYAoKIyMjIFBsb3QgdGhlIEdBTSBtb2RlbCBmaXRzCgpgYGB7cn0KeW1pbiA9IC00MAp5bWF4ID0gMTAwCnhtaW4gPSAwCnhtYXggPSAyMAoKZ2V0X2RfcG9pbnRzID0gZnVuY3Rpb24oZGYsIG1vZGVsLCB0cmFpbmluZywgY29ycHVzKXsKICB4ID0gZGVuc2l0eShkZiRzdXJwcmlzYWwpJHgKICB5ID0gZGVuc2l0eShkZiRzdXJwcmlzYWwpJHkKICByZXR1cm4oZGF0YS5mcmFtZShtb2RlbCwgdHJhaW5pbmcsIGNvcnB1cywgeCwgeSkpCn0KCiMgR2V0IHRoZSBkZW5zaXR5IHBvaW50cwpkZW5zaXR5X2RhdGEgPSBhbGxfZGF0YSAlPiUKICBtdXRhdGUobW9kZWwgPSByZWNvZGUobW9kZWwsIHZhbmlsbGE9ImxzdG0iKSkgJT4lCiAgZ3JvdXBfYnkobW9kZWwsIHRyYWluaW5nLCBjb3JwdXMpICU+JQogICAgZG8oe2dldF9kX3BvaW50cyguLCB1bmlxdWUoLiRtb2RlbCksIHVuaXF1ZSguJHRyYWluaW5nKSwgdW5pcXVlKC4kY29ycHVzKSl9KSAlPiUKICB1bmdyb3VwKCkgJT4lCiAgbXV0YXRlKHRyYWluaW5nX3ZvY2FiPWFzLmZhY3RvcihpZmVsc2Uoc3RyX2RldGVjdCh0cmFpbmluZywgImdwdGJwZSIpLCAiZ3B0YnBlIiwgYXMuY2hhcmFjdGVyKHRyYWluaW5nKSkpLAogICAgICAgICB0cmFpbmluZ19zb3VyY2U9YXMuZmFjdG9yKHN0cl9yZXBsYWNlKGFzLmNoYXJhY3Rlcih0cmFpbmluZyksICItZ3B0YnBlIiwgIiIpKSwKICAgICAgICAgYnBlPXRyYWluaW5nX3ZvY2FiID09ICJncHRicGUiLCkgJT4lIAogIGZpbHRlcih4PjAsIHg8MjApCgpzbW9vdGhzID0gcmVhZC5jc3YoIi4uL2RhdGEvZ2FtX3Ntb290aHMuc2wyMDEzLmFsbC5jc3YiKQpnYW1fc21vb3RocyA9IHNtb290aHMgJT4lIAogICMgUGxvdCBuaWNldGllcwogICMgdmFuaWxsYSAtPiBsc3RtCiAgbXV0YXRlKG1vZGVsPXJlY29kZShtb2RlbCwgdmFuaWxsYT0ibHN0bSIpKSAlPiUgCiAgIyBDcmVhdGUgQlBFIHZzIG5vbi1CUEUgdmFyaWFibGUKICBtdXRhdGUoYnBlPXRyYWluaW5nX3ZvY2FiID09ICJncHRicGUiKSAlPiUgCiAgCiAgIyBGaXggMCBzdXJwcmlzYWwgPSAwIG1zCiAgZ3JvdXBfYnkodHJhaW5pbmdfdm9jYWIsIG1vZGVsLCBjb3JwdXMpICU+JSAKICAgIG11dGF0ZShkZWx0YT0wIC0geVsxXSwKICAgICAgICAgICAjIFRyaW0gbG93ZXIgYm91bmQgdG8gbWFrZSBzdXJlIGl0IGdldHMgcGxvdHRlZCB3aXRoaW4gdGhlIHlsaW0KICAgICAgICAgICB5X2xvd2VyPXBtYXgoeW1pbiwgeV9sb3dlciArIGRlbHRhKSwKICAgICAgICAgICB5PXkgKyBkZWx0YSwKICAgICAgICAgICAjIFRyaW0gdXBwZXIgYm91bmQgbGlrZXdpc2UKICAgICAgICAgICB5X3VwcGVyPXBtaW4oeW1heCwgeV91cHBlciArIGRlbHRhKSkgJT4lIAogIHVuZ3JvdXAoKQoKZGVuc2l0eV9kYXRhICU+JSBmaWx0ZXIobW9kZWwgPT0gImdwdDIiLCB0cmFpbmluZ19zb3VyY2UgPT0gImJsbGlwLWxnIikKCmdncGxvdCgpICsKICB0aGVtZV9idygpICsKICAgIGFubm90YXRlKCJyZWN0IiwgeG1pbj0wLCB4bWF4PTIwLCB5bWluPS00MCx5bWF4PS0xNSwgZmlsbD0iZ3JleSIsIGFscGhhPTAuMykgKwogICAgZ2VvbV9saW5lKGRhdGEgPSBkZW5zaXR5X2RhdGEsIGFlcyh4PXgsIHk9eSoyMDAgLSA0MCwgbGluZXR5cGU9YnBlKSwgY29sb3I9ImdyZXkiKSArICMsIHNpemUgPSAwLjEpICsjIDAuNSkgKwogICAgZ2VvbV9saW5lKGRhdGEgPSBnYW1fc21vb3RocywgYWVzKHg9c3VycHJpc2FsLCB5PXksIGxpbmV0eXBlPWJwZSwgY29sb3I9dHJhaW5pbmdfc291cmNlKSwgc2l6ZT0wLjUpICsKICAgIGdlb21fcmliYm9uKGRhdGEgPSBnYW1fc21vb3RocywgYWVzKHg9c3VycHJpc2FsLCB5bWluPXlfbG93ZXIsIHltYXg9eV91cHBlciwgZmlsbD10cmFpbmluZ19zb3VyY2UsIGNvbG9yPU5BLCBsaW5ldHlwZT1icGUpLCBhbHBoYT0wLjMpICsKICAgIGZhY2V0X2dyaWQoY29ycHVzIH4gdHJhaW5pbmdfc291cmNlICsgbW9kZWwsIHNjYWxlcz0iZnJlZSIpICsKICAgIHlsaW0oeW1pbiwgeW1heCkgKwogICAgY29vcmRfY2FydGVzaWFuKHhsaW09YygwLDIwKSkgKwogICAgc2NhbGVfY29sb3JfbWFudWFsKHZhbHVlcyA9IGMoImJsbGlwLWxnIj0iIzQ0MDE1NEZGIiwgImJsbGlwLW1kIj0iIzM5NTY4Q0ZGIiwgImJsbGlwLXNtIj0iIzFGOTY4QkZGIiwgImJsbGlwLXhzIj0iIzczRDA1NUZGIiwgImdwdGJwZSI9IiNmMDk0MWYiKSkgKwogICAgc2NhbGVfZmlsbF9tYW51YWwodmFsdWVzID0gYygiYmxsaXAtbGciPSIjNDQwMTU0RkYiLCAiYmxsaXAtbWQiPSIjMzk1NjhDRkYiLCAgImJsbGlwLXNtIj0iIzFGOTY4QkZGIiwgICJibGxpcC14cyI9IiM3M0QwNTVGRiIsICJncHRicGUiPSIjZjA5NDFmIiksIGd1aWRlPSJub25lIikgKwogICAgc2NhbGVfeF9jb250aW51b3VzKGxhYmVscz1jKDAsIDEwLCAyMCksIGJyZWFrcz1jKDAsIDEwLCAyMCksIG1pbm9yX2JyZWFrcyA9IE5VTEwpICsKICAgIHRoZW1lKGxlZ2VuZC5wb3NpdGlvbiA9ICJib3R0b20iKSArCiAgICBsYWJzKHg9IlN1cnByaXNhbCAoYml0cykiLCB5PSJTbG93ZG93biBkdWUgdG8gc3VycHJpc2FsIChtcykiLAogICAgICAgICBsaW5ldHlwZT0iQlBFIiwKICAgICAgICAgY29sb3I9IlRyYWluaW5nIGRhdGEiKQogZ2dzYXZlKCIuLi9pbWFnZXMvY29nc2NpMjAyMC9nYW1fc3VycF9jb3JyX2Z1bGwucGRmIiwgaGVpZ2h0PTUuNSx3aWR0aD0xMiwgZGV2aWNlID0gY2Fpcm9fcGRmKQoKYGBgCgoKCgoKCgo=